generic-library/bionic
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

libc: optimize pthread mutex lock/unlock operations (1/2)

Browse Source 
This patch provides several small optimizations to the
implementation of mutex locking and unlocking. Note that
a following patch will get rid of the global recursion
lock, and provide a few more aggressive changes, I
though it'd be simpler to split this change in two parts.

+ New behaviour: pthread_mutex_lock et al now detect
  recursive mutex overflows and will return EAGAIN in
  this case, as suggested by POSIX. Before, the counter
  would just wrap to 0.

- Remove un-necessary reloads of the mutex value from memory
  by storing it in a local variable (mvalue)

- Remove un-necessary reload of the mutex value by passing
  the 'shared' local variable to _normal_lock / _normal_unlock

- Remove un-necessary reload of the mutex value by using a
  new macro (MUTEX_VALUE_OWNER()) to compare the thread id
  for recursive/errorcheck mutexes

- Use a common inlined function to increment the counter
  of a recursive mutex. Also do not use the global
  recursion lock in this case to speed it up.

Change-Id: I106934ec3a8718f8f852ef547f3f0e9d9435c816
This commit is contained in:
David 'Digit' Turner
2011-12-07 14:02:17 +01:00
parent 6c6de44f04
commit 022d303116
1 changed files with 86 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

169
libc/bionic/pthread.c
View File

@@ -760,8 +760,11 @@ int pthread_setschedparam(pthread_t thid, int policy,
*/ */
#define MUTEX_OWNER(m) (((m)->value >> 16) & 0xffff) #define MUTEX_VALUE_OWNER(v) (((v) >> 16) & 0xffff)
#define MUTEX_COUNTER(m) (((m)->value >> 2) & 0xfff) #define MUTEX_VALUE_COUNTER(v) (((v) >> 2) & 0xfff)
#define MUTEX_OWNER(m) MUTEX_VALUE_OWNER((m)->value)
#define MUTEX_COUNTER(m) MUTEX_VALUE_COUNTER((m)->value)
#define MUTEX_TYPE_MASK 0xc000 #define MUTEX_TYPE_MASK 0xc000
#define MUTEX_TYPE_NORMAL 0x0000 #define MUTEX_TYPE_NORMAL 0x0000
@@ -922,10 +925,8 @@ int pthread_mutex_destroy(pthread_mutex_t *mutex)
* the lock state field. * the lock state field.
*/ */
static __inline__ void static __inline__ void
_normal_lock(pthread_mutex_t* mutex) _normal_lock(pthread_mutex_t* mutex, int shared)
{ {
/* We need to preserve the shared flag during operations */
int shared = mutex->value & MUTEX_SHARED_MASK;
/* /*
* The common case is an unlocked mutex, so we begin by trying to * The common case is an unlocked mutex, so we begin by trying to
* change the lock's state from 0 to 1. __bionic_cmpxchg() returns 0 * change the lock's state from 0 to 1. __bionic_cmpxchg() returns 0
@@ -960,13 +961,10 @@ _normal_lock(pthread_mutex_t* mutex)
* that we are in fact the owner of this lock. * that we are in fact the owner of this lock.
*/ */
static __inline__ void static __inline__ void
_normal_unlock(pthread_mutex_t* mutex) _normal_unlock(pthread_mutex_t* mutex, int shared)
{ {
ANDROID_MEMBAR_FULL(); ANDROID_MEMBAR_FULL();
/* We need to preserve the shared flag during operations */
int shared = mutex->value & MUTEX_SHARED_MASK;
/* /*
* The mutex state will be 1 or (rarely) 2. We use an atomic decrement * The mutex state will be 1 or (rarely) 2. We use an atomic decrement
* to release the lock. __bionic_atomic_dec() returns the previous value; * to release the lock. __bionic_atomic_dec() returns the previous value;
@@ -1014,58 +1012,88 @@ _normal_unlock(pthread_mutex_t* mutex)
} }
} }
static pthread_mutex_t __recursive_lock = PTHREAD_MUTEX_INITIALIZER; /* This common inlined function is used to increment the counter of an
* errorcheck or recursive mutex.
static void *
_recursive_lock(void) * For errorcheck mutexes, it will return EDEADLK
* If the counter overflows, it will return EAGAIN
* Otherwise, it atomically increments the counter and returns 0
* after providing an acquire barrier.
*
* mtype is the current mutex type
* mvalue is the current mutex value (already loaded)
* mutex pointers to the mutex.
*/
static __inline__ __attribute__((always_inline)) int
_recursive_increment(pthread_mutex_t* mutex, int mvalue, int mtype)
{ {
_normal_lock(&__recursive_lock); if (mtype == MUTEX_TYPE_ERRORCHECK) {
/* trying to re-lock a mutex we already acquired */
return EDEADLK;
} }
static void /* Detect recursive lock overflow and return EAGAIN.
* This is safe because only the owner thread can modify the
* counter bits in the mutes value
*/
if ((mvalue & MUTEX_COUNTER_MASK) == MUTEX_COUNTER_MASK) {
return EAGAIN;
}
/* We own the mutex, but other threads are able to change
* the contents (e.g. promoting it to "contended" by changing
* its lower bits), so we need to atomically update it using
* a cmpxchg loop.
*/
for (;;) {
/* increment counter, overflow was already checked */
int newvalue = mvalue + (1 << MUTEX_COUNTER_SHIFT);
if (__likely(__bionic_cmpxchg(mvalue, newvalue, &mutex->value) == 0)) {
ANDROID_MEMBAR_FULL();
return 0;
}
/* argh, value was changed, by another thread which updated the 'state'
* field, so reload and try again.
*/
mvalue = mutex->value;
}
}
static pthread_mutex_t __recursive_lock = PTHREAD_MUTEX_INITIALIZER;
static __inline__ void
_recursive_lock(void)
{
_normal_lock(&__recursive_lock, 0);
}
static __inline__ void
_recursive_unlock(void) _recursive_unlock(void)
{ {
_normal_unlock(&__recursive_lock ); _normal_unlock(&__recursive_lock, 0);
} }
int pthread_mutex_lock(pthread_mutex_t *mutex) int pthread_mutex_lock(pthread_mutex_t *mutex)
{ {
int mtype, tid, new_lock_type, shared; int mvalue, mtype, tid, new_lock_type, shared;
if (__unlikely(mutex == NULL)) if (__unlikely(mutex == NULL))
return EINVAL; return EINVAL;
mtype = (mutex->value & MUTEX_TYPE_MASK); mvalue = mutex->value;
shared = (mutex->value & MUTEX_SHARED_MASK); mtype = (mvalue & MUTEX_TYPE_MASK);
shared = (mvalue & MUTEX_SHARED_MASK);
/* Handle normal case first */ /* Handle normal case first */
if ( __likely(mtype == MUTEX_TYPE_NORMAL) ) { if ( __likely(mtype == MUTEX_TYPE_NORMAL) ) {
_normal_lock(mutex); _normal_lock(mutex, shared);
return 0; return 0;
} }
/* Do we already own this recursive or error-check mutex ? */ /* Do we already own this recursive or error-check mutex ? */
tid = __get_thread()->kernel_id; tid = __get_thread()->kernel_id;
if ( tid == MUTEX_OWNER(mutex) ) if ( tid == MUTEX_VALUE_OWNER(mvalue) )
{ return _recursive_increment(mutex, mvalue, mtype);
int oldv, counter;
if (mtype == MUTEX_TYPE_ERRORCHECK) {
/* trying to re-lock a mutex we already acquired */
return EDEADLK;
}
/*
* We own the mutex, but other threads are able to change
* the contents (e.g. promoting it to "contended"), so we
* need to hold the global lock.
*/
_recursive_lock();
oldv = mutex->value;
counter = (oldv + (1 << MUTEX_COUNTER_SHIFT)) & MUTEX_COUNTER_MASK;
mutex->value = (oldv & ~MUTEX_COUNTER_MASK) | counter;
_recursive_unlock();
return 0;
}
/* We don't own the mutex, so try to get it. /* We don't own the mutex, so try to get it.
* *
@@ -1109,23 +1137,24 @@ int pthread_mutex_lock(pthread_mutex_t *mutex)
int pthread_mutex_unlock(pthread_mutex_t *mutex) int pthread_mutex_unlock(pthread_mutex_t *mutex)
{ {
int mtype, tid, oldv, shared; int mvalue, mtype, tid, oldv, shared;
if (__unlikely(mutex == NULL)) if (__unlikely(mutex == NULL))
return EINVAL; return EINVAL;
mtype = (mutex->value & MUTEX_TYPE_MASK); mvalue = mutex->value;
shared = (mutex->value & MUTEX_SHARED_MASK); mtype = (mvalue & MUTEX_TYPE_MASK);
shared = (mvalue & MUTEX_SHARED_MASK);
/* Handle common case first */ /* Handle common case first */
if (__likely(mtype == MUTEX_TYPE_NORMAL)) { if (__likely(mtype == MUTEX_TYPE_NORMAL)) {
_normal_unlock(mutex); _normal_unlock(mutex, shared);
return 0; return 0;
} }
/* Do we already own this recursive or error-check mutex ? */ /* Do we already own this recursive or error-check mutex ? */
tid = __get_thread()->kernel_id; tid = __get_thread()->kernel_id;
if ( tid != MUTEX_OWNER(mutex) ) if ( tid != MUTEX_VALUE_OWNER(mvalue) )
return EPERM; return EPERM;
/* We do, decrement counter or release the mutex if it is 0 */ /* We do, decrement counter or release the mutex if it is 0 */
@@ -1149,13 +1178,14 @@ int pthread_mutex_unlock(pthread_mutex_t *mutex)
int pthread_mutex_trylock(pthread_mutex_t *mutex) int pthread_mutex_trylock(pthread_mutex_t *mutex)
{ {
int mtype, tid, oldv, shared; int mvalue, mtype, tid, oldv, shared;
if (__unlikely(mutex == NULL)) if (__unlikely(mutex == NULL))
return EINVAL; return EINVAL;
mtype = (mutex->value & MUTEX_TYPE_MASK); mvalue = mutex->value;
shared = (mutex->value & MUTEX_SHARED_MASK); mtype = (mvalue & MUTEX_TYPE_MASK);
shared = (mvalue & MUTEX_SHARED_MASK);
/* Handle common case first */ /* Handle common case first */
if ( __likely(mtype == MUTEX_TYPE_NORMAL) ) if ( __likely(mtype == MUTEX_TYPE_NORMAL) )
@@ -1170,22 +1200,8 @@ int pthread_mutex_trylock(pthread_mutex_t *mutex)
/* Do we already own this recursive or error-check mutex ? */ /* Do we already own this recursive or error-check mutex ? */
tid = __get_thread()->kernel_id; tid = __get_thread()->kernel_id;
if ( tid == MUTEX_OWNER(mutex) ) if ( tid == MUTEX_VALUE_OWNER(mvalue) )
{ return _recursive_increment(mutex, mvalue, mtype);
int counter;
if (mtype == MUTEX_TYPE_ERRORCHECK) {
/* already locked by ourselves */
return EDEADLK;
}
_recursive_lock();
oldv = mutex->value;
counter = (oldv + (1 << MUTEX_COUNTER_SHIFT)) & MUTEX_COUNTER_MASK;
mutex->value = (oldv & ~MUTEX_COUNTER_MASK) | counter;
_recursive_unlock();
return 0;
}
/* Restore sharing bit in mtype */ /* Restore sharing bit in mtype */
mtype |= shared; mtype |= shared;
@@ -1243,7 +1259,7 @@ int pthread_mutex_lock_timeout_np(pthread_mutex_t *mutex, unsigned msecs)
clockid_t clock = CLOCK_MONOTONIC; clockid_t clock = CLOCK_MONOTONIC;
struct timespec abstime; struct timespec abstime;
struct timespec ts; struct timespec ts;
int mtype, tid, oldv, new_lock_type, shared; int mvalue, mtype, tid, oldv, new_lock_type, shared;
/* compute absolute expiration time */ /* compute absolute expiration time */
__timespec_to_relative_msec(&abstime, msecs, clock); __timespec_to_relative_msec(&abstime, msecs, clock);
@@ -1251,8 +1267,9 @@ int pthread_mutex_lock_timeout_np(pthread_mutex_t *mutex, unsigned msecs)
if (__unlikely(mutex == NULL)) if (__unlikely(mutex == NULL))
return EINVAL; return EINVAL;
mtype = (mutex->value & MUTEX_TYPE_MASK); mvalue = mutex->value;
shared = (mutex->value & MUTEX_SHARED_MASK); mtype = (mvalue & MUTEX_TYPE_MASK);
shared = (mvalue & MUTEX_SHARED_MASK);
/* Handle common case first */ /* Handle common case first */
if ( __likely(mtype == MUTEX_TYPE_NORMAL) ) if ( __likely(mtype == MUTEX_TYPE_NORMAL) )
@@ -1276,22 +1293,8 @@ int pthread_mutex_lock_timeout_np(pthread_mutex_t *mutex, unsigned msecs)
/* Do we already own this recursive or error-check mutex ? */ /* Do we already own this recursive or error-check mutex ? */
tid = __get_thread()->kernel_id; tid = __get_thread()->kernel_id;
if ( tid == MUTEX_OWNER(mutex) ) if ( tid == MUTEX_VALUE_OWNER(mvalue) )
{ return _recursive_increment(mutex, mvalue, mtype);
int oldv, counter;
if (mtype == MUTEX_TYPE_ERRORCHECK) {
/* already locked by ourselves */
return EDEADLK;
}
_recursive_lock();
oldv = mutex->value;
counter = (oldv + (1 << MUTEX_COUNTER_SHIFT)) & MUTEX_COUNTER_MASK;
mutex->value = (oldv & ~MUTEX_COUNTER_MASK) | counter;
_recursive_unlock();
return 0;
}
/* We don't own the mutex, so try to get it. /* We don't own the mutex, so try to get it.
* *