generic-library/bionic
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

am c210e84f: am e1edd301: Merge "Refactor pthread_key.cpp to be lock-free."

Browse Source 
* commit 'c210e84f28633d15f84a66a0652a4485c5fbc24d':
  Refactor pthread_key.cpp to be lock-free.
This commit is contained in:
Yabin Cui
2015-03-04 02:00:43 +00:00
committed by Android Git Automerger
parent 92cadb19fb c210e84f28
commit 01351a7289
4 changed files with 130 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
libc/bionic/pthread_create.cpp
View File

@@ -56,7 +56,8 @@ void __init_tls(pthread_internal_t* thread) {
if (thread->mmap_size == 0) { if (thread->mmap_size == 0) {
// If the TLS area was not allocated by mmap(), it may not have been cleared to zero. // If the TLS area was not allocated by mmap(), it may not have been cleared to zero.
// So assume the worst and zero the TLS area. // So assume the worst and zero the TLS area.
memset(&thread->tls[0], 0, BIONIC_TLS_SLOTS * sizeof(void*)); memset(thread->tls, 0, sizeof(thread->tls));
memset(thread->key_data, 0, sizeof(thread->key_data));
} }
// Slot 0 must point to itself. The x86 Linux kernel reads the TLS from %fs:0. // Slot 0 must point to itself. The x86 Linux kernel reads the TLS from %fs:0.
@@ -155,7 +156,7 @@ static int __allocate_thread(pthread_attr_t* attr, pthread_internal_t** threadp,
} }
// Mapped space(or user allocated stack) is used for: // Mapped space(or user allocated stack) is used for:
// thread_internal_t (including tls array) // thread_internal_t
// thread stack (including guard page) // thread stack (including guard page)
stack_top -= sizeof(pthread_internal_t); stack_top -= sizeof(pthread_internal_t);
pthread_internal_t* thread = reinterpret_cast<pthread_internal_t*>(stack_top); pthread_internal_t* thread = reinterpret_cast<pthread_internal_t*>(stack_top);

7
libc/bionic/pthread_internal.h
View File

@@ -44,6 +44,11 @@
/* Is this the main thread? */ /* Is this the main thread? */
#define PTHREAD_ATTR_FLAG_MAIN_THREAD 0x80000000 #define PTHREAD_ATTR_FLAG_MAIN_THREAD 0x80000000
struct pthread_key_data_t {
uintptr_t seq; // Use uintptr_t just for alignment, as we use pointer below.
void* data;
};
struct pthread_internal_t { struct pthread_internal_t {
struct pthread_internal_t* next; struct pthread_internal_t* next;
struct pthread_internal_t* prev; struct pthread_internal_t* prev;
@@ -86,6 +91,8 @@ struct pthread_internal_t {
void* tls[BIONIC_TLS_SLOTS]; void* tls[BIONIC_TLS_SLOTS];
pthread_key_data_t key_data[BIONIC_PTHREAD_KEY_COUNT];
/* /*
* The dynamic linker implements dlerror(3), which makes it hard for us to implement this * The dynamic linker implements dlerror(3), which makes it hard for us to implement this
* per-thread buffer by simply using malloc(3) and free(3). * per-thread buffer by simply using malloc(3) and free(3).

255
libc/bionic/pthread_key.cpp
View File

@@ -28,175 +28,98 @@
#include <errno.h> #include <errno.h>
#include <pthread.h> #include <pthread.h>
#include <stdatomic.h>
#include "private/bionic_tls.h" #include "private/bionic_tls.h"
#include "pthread_internal.h" #include "pthread_internal.h"
/* A technical note regarding our thread-local-storage (TLS) implementation:
*
* There can be up to BIONIC_TLS_SLOTS independent TLS keys in a given process,
* The keys below TLS_SLOT_FIRST_USER_SLOT are reserved for Bionic to hold
* special thread-specific variables like errno or a pointer to
* the current thread's descriptor. These entries cannot be accessed through
* pthread_getspecific() / pthread_setspecific() or pthread_key_delete()
*
* The 'tls_map_t' type defined below implements a shared global map of
* currently created/allocated TLS keys and the destructors associated
* with them.
*
* The global TLS map simply contains a bitmap of allocated keys, and
* an array of destructors.
*
* Each thread has a TLS area that is a simple array of BIONIC_TLS_SLOTS void*
* pointers. the TLS area of the main thread is stack-allocated in
* __libc_init_common, while the TLS area of other threads is placed at
* the top of their stack in pthread_create.
*
* When pthread_key_delete() is called it will erase the key's bitmap bit
* and its destructor, and will also clear the key data in the TLS area of
* all created threads. As mandated by Posix, it is the responsibility of
* the caller of pthread_key_delete() to properly reclaim the objects that
* were pointed to by these data fields (either before or after the call).
*/
#define TLSMAP_BITS 32
#define TLSMAP_WORDS ((BIONIC_TLS_SLOTS+TLSMAP_BITS-1)/TLSMAP_BITS)
#define TLSMAP_WORD(m,k) (m).map[(k)/TLSMAP_BITS]
#define TLSMAP_MASK(k) (1U << ((k)&(TLSMAP_BITS-1)))
static inline bool IsValidUserKey(pthread_key_t key) {
return (key >= TLS_SLOT_FIRST_USER_SLOT && key < BIONIC_TLS_SLOTS);
}
typedef void (*key_destructor_t)(void*); typedef void (*key_destructor_t)(void*);
struct tls_map_t { #define SEQ_KEY_IN_USE_BIT 0
bool is_initialized;
/* bitmap of allocated keys */ #define SEQ_INCREMENT_STEP (1 << SEQ_KEY_IN_USE_BIT)
uint32_t map[TLSMAP_WORDS];
key_destructor_t key_destructors[BIONIC_TLS_SLOTS]; // pthread_key_internal_t records the use of each pthread key slot:
// seq records the state of the slot.
// bit 0 is 1 when the key is in use, 0 when it is unused. Each time we create or delete the
// pthread key in the slot, we increse the seq by 1 (which inverts bit 0). The reason to use
// a sequence number instead of a boolean value here is that when the key slot is deleted and
// reused for a new key, pthread_getspecific will not return stale data.
// key_destructor records the destructor called at thread exit.
struct pthread_key_internal_t {
atomic_uintptr_t seq;
atomic_uintptr_t key_destructor;
}; };
class ScopedTlsMapAccess { static pthread_key_internal_t key_map[BIONIC_PTHREAD_KEY_COUNT];
public:
ScopedTlsMapAccess() {
Lock();
// If this is the first time the TLS map has been accessed, static inline bool SeqOfKeyInUse(uintptr_t seq) {
// mark the slots belonging to well-known keys as being in use. return seq & (1 << SEQ_KEY_IN_USE_BIT);
// This isn't currently necessary because the well-known keys }
// can only be accessed directly by bionic itself, do not have
// destructors, and all the functions that touch the TLS map
// start after the maximum well-known slot.
if (!s_tls_map_.is_initialized) {
for (pthread_key_t key = 0; key < TLS_SLOT_FIRST_USER_SLOT; ++key) {
SetInUse(key, NULL);
}
s_tls_map_.is_initialized = true;
}
}
~ScopedTlsMapAccess() { static inline bool KeyInValidRange(pthread_key_t key) {
Unlock(); return key >= 0 && key < BIONIC_PTHREAD_KEY_COUNT;
} }
int CreateKey(pthread_key_t* result, void (*key_destructor)(void*)) { // Called from pthread_exit() to remove all pthread keys. This must call the destructor of
// Take the first unallocated key. // all keys that have a non-NULL data value and a non-NULL destructor.
for (int key = 0; key < BIONIC_TLS_SLOTS; ++key) { __LIBC_HIDDEN__ void pthread_key_clean_all() {
if (!IsInUse(key)) { // Because destructors can do funky things like deleting/creating other keys,
SetInUse(key, key_destructor); // we need to implement this in a loop.
*result = key; pthread_key_data_t* key_data = __get_thread()->key_data;
return 0; for (size_t rounds = PTHREAD_DESTRUCTOR_ITERATIONS; rounds > 0; --rounds) {
}
}
// We hit PTHREAD_KEYS_MAX. POSIX says EAGAIN for this case.
return EAGAIN;
}
void DeleteKey(pthread_key_t key) {
TLSMAP_WORD(s_tls_map_, key) &= ~TLSMAP_MASK(key);
s_tls_map_.key_destructors[key] = NULL;
}
bool IsInUse(pthread_key_t key) {
return (TLSMAP_WORD(s_tls_map_, key) & TLSMAP_MASK(key)) != 0;
}
void SetInUse(pthread_key_t key, void (*key_destructor)(void*)) {
TLSMAP_WORD(s_tls_map_, key) |= TLSMAP_MASK(key);
s_tls_map_.key_destructors[key] = key_destructor;
}
// Called from pthread_exit() to remove all TLS key data
// from this thread's TLS area. This must call the destructor of all keys
// that have a non-NULL data value and a non-NULL destructor.
void CleanAll() {
void** tls = __get_tls();
// Because destructors can do funky things like deleting/creating other
// keys, we need to implement this in a loop.
for (int rounds = PTHREAD_DESTRUCTOR_ITERATIONS; rounds > 0; --rounds) {
size_t called_destructor_count = 0; size_t called_destructor_count = 0;
for (int key = 0; key < BIONIC_TLS_SLOTS; ++key) { for (size_t i = 0; i < BIONIC_PTHREAD_KEY_COUNT; ++i) {
if (IsInUse(key)) { uintptr_t seq = atomic_load_explicit(&key_map[i].seq, memory_order_relaxed);
void* data = tls[key]; if (SeqOfKeyInUse(seq) && seq == key_data[i].seq && key_data[i].data != NULL) {
void (*key_destructor)(void*) = s_tls_map_.key_destructors[key]; // Other threads may be calling pthread_key_delete/pthread_key_create while current thread
// is exiting. So we need to ensure we read the right key_destructor.
// We can rely on a user-established happens-before relationship between the creation and
// use of pthread key to ensure that we're not getting an earlier key_destructor.
// To avoid using the key_destructor of the newly created key in the same slot, we need to
// recheck the sequence number after reading key_destructor. As a result, we either see the
// right key_destructor, or the sequence number must have changed when we reread it below.
key_destructor_t key_destructor = reinterpret_cast<key_destructor_t>(
atomic_load_explicit(&key_map[i].key_destructor, memory_order_relaxed));
if (key_destructor == NULL) {
continue;
}
atomic_thread_fence(memory_order_acquire);
if (atomic_load_explicit(&key_map[i].seq, memory_order_relaxed) != seq) {
continue;
}
if (data != NULL && key_destructor != NULL) { // We need to clear the key data now, this will prevent the destructor (or a later one)
// we need to clear the key data now, this will prevent the // from seeing the old value if it calls pthread_getspecific().
// destructor (or a later one) from seeing the old value if // We don't do this if 'key_destructor == NULL' just in case another destructor
// it calls pthread_getspecific() for some odd reason // function is responsible for manually releasing the corresponding data.
void* data = key_data[i].data;
key_data[i].data = NULL;
// we do not do this if 'key_destructor == NULL' just in case another
// destructor function might be responsible for manually
// releasing the corresponding data.
tls[key] = NULL;
// because the destructor is free to call pthread_key_create
// and/or pthread_key_delete, we need to temporarily unlock
// the TLS map
Unlock();
(*key_destructor)(data); (*key_destructor)(data);
Lock();
++called_destructor_count; ++called_destructor_count;
} }
} }
}
// If we didn't call any destructors, there is no need to check the TLS data again. // If we didn't call any destructors, there is no need to check the pthread keys again.
if (called_destructor_count == 0) { if (called_destructor_count == 0) {
break; break;
} }
} }
}
private:
static tls_map_t s_tls_map_;
static pthread_mutex_t s_tls_map_lock_;
void Lock() {
pthread_mutex_lock(&s_tls_map_lock_);
}
void Unlock() {
pthread_mutex_unlock(&s_tls_map_lock_);
}
};
__LIBC_HIDDEN__ tls_map_t ScopedTlsMapAccess::s_tls_map_;
__LIBC_HIDDEN__ pthread_mutex_t ScopedTlsMapAccess::s_tls_map_lock_;
__LIBC_HIDDEN__ void pthread_key_clean_all() {
ScopedTlsMapAccess tls_map;
tls_map.CleanAll();
} }
int pthread_key_create(pthread_key_t* key, void (*key_destructor)(void*)) { int pthread_key_create(pthread_key_t* key, void (*key_destructor)(void*)) {
ScopedTlsMapAccess tls_map; for (size_t i = 0; i < BIONIC_PTHREAD_KEY_COUNT; ++i) {
return tls_map.CreateKey(key, key_destructor); uintptr_t seq = atomic_load_explicit(&key_map[i].seq, memory_order_relaxed);
while (!SeqOfKeyInUse(seq)) {
if (atomic_compare_exchange_weak(&key_map[i].seq, &seq, seq + SEQ_INCREMENT_STEP)) {
atomic_store(&key_map[i].key_destructor, reinterpret_cast<uintptr_t>(key_destructor));
*key = i;
return 0;
}
}
}
return EAGAIN;
} }
// Deletes a pthread_key_t. note that the standard mandates that this does // Deletes a pthread_key_t. note that the standard mandates that this does
@@ -204,42 +127,44 @@ int pthread_key_create(pthread_key_t* key, void (*key_destructor)(void*)) {
// responsibility of the caller to properly dispose of the corresponding data // responsibility of the caller to properly dispose of the corresponding data
// and resources, using any means it finds suitable. // and resources, using any means it finds suitable.
int pthread_key_delete(pthread_key_t key) { int pthread_key_delete(pthread_key_t key) {
ScopedTlsMapAccess tls_map; if (!KeyInValidRange(key)) {
if (!IsValidUserKey(key) || !tls_map.IsInUse(key)) {
return EINVAL; return EINVAL;
} }
// Increase seq to invalidate values in all threads.
// Clear value in all threads. uintptr_t seq = atomic_load_explicit(&key_map[key].seq, memory_order_relaxed);
pthread_mutex_lock(&g_thread_list_lock); if (SeqOfKeyInUse(seq)) {
for (pthread_internal_t* t = g_thread_list; t != NULL; t = t->next) { if (atomic_compare_exchange_strong(&key_map[key].seq, &seq, seq + SEQ_INCREMENT_STEP)) {
t->tls[key] = NULL;
}
tls_map.DeleteKey(key);
pthread_mutex_unlock(&g_thread_list_lock);
return 0; return 0;
}
}
return EINVAL;
} }
void* pthread_getspecific(pthread_key_t key) { void* pthread_getspecific(pthread_key_t key) {
if (!IsValidUserKey(key)) { if (!KeyInValidRange(key)) {
return NULL; return NULL;
} }
uintptr_t seq = atomic_load_explicit(&key_map[key].seq, memory_order_relaxed);
// For performance reasons, we do not lock/unlock the global TLS map pthread_key_data_t* data = &(__get_thread()->key_data[key]);
// to check that the key is properly allocated. If the key was not // It is user's responsibility to synchornize between the creation and use of pthread keys,
// allocated, the value read from the TLS should always be NULL // so we use memory_order_relaxed when checking the sequence number.
// due to pthread_key_delete() clearing the values for all threads. if (__predict_true(SeqOfKeyInUse(seq) && data->seq == seq)) {
return __get_tls()[key]; return data->data;
}
data->data = NULL;
return NULL;
} }
int pthread_setspecific(pthread_key_t key, const void* ptr) { int pthread_setspecific(pthread_key_t key, const void* ptr) {
ScopedTlsMapAccess tls_map; if (!KeyInValidRange(key)) {
if (!IsValidUserKey(key) || !tls_map.IsInUse(key)) {
return EINVAL; return EINVAL;
} }
uintptr_t seq = atomic_load_explicit(&key_map[key].seq, memory_order_relaxed);
__get_tls()[key] = const_cast<void*>(ptr); if (SeqOfKeyInUse(seq)) {
pthread_key_data_t* data = &(__get_thread()->key_data[key]);
data->seq = seq;
data->data = const_cast<void*>(ptr);
return 0; return 0;
}
return EINVAL;
} }

35
libc/private/bionic_tls.h
View File

@@ -67,15 +67,15 @@ enum {
TLS_SLOT_STACK_GUARD = 5, // GCC requires this specific slot for x86. TLS_SLOT_STACK_GUARD = 5, // GCC requires this specific slot for x86.
TLS_SLOT_DLERROR, TLS_SLOT_DLERROR,
TLS_SLOT_FIRST_USER_SLOT // Must come last! BIONIC_TLS_SLOTS // Must come last!
}; };
/* /*
* There are two kinds of slot used internally by bionic --- there are the well-known slots * Bionic uses some pthread keys internally. All pthread keys used internally
* enumerated above, and then there are those that are allocated during startup by calls to * should be created in constructors.
* pthread_key_create; grep for GLOBAL_INIT_THREAD_LOCAL_BUFFER to find those. We need to manually * We need to manually maintain the count of pthread keys used internally, but
* maintain that second number, but pthread_test will fail if we forget. * pthread_test should fail if we forget.
* Following are current pthread keys used internally: * Following are current pthread keys used internally by libc:
* basename libc (GLOBAL_INIT_THREAD_LOCAL_BUFFER) * basename libc (GLOBAL_INIT_THREAD_LOCAL_BUFFER)
* dirname libc (GLOBAL_INIT_THREAD_LOCAL_BUFFER) * dirname libc (GLOBAL_INIT_THREAD_LOCAL_BUFFER)
* uselocale libc * uselocale libc
@@ -88,28 +88,29 @@ enum {
* passwd libc (GLOBAL_INIT_THREAD_LOCAL_BUFFER) * passwd libc (GLOBAL_INIT_THREAD_LOCAL_BUFFER)
* group libc (GLOBAL_INIT_THREAD_LOCAL_BUFFER) * group libc (GLOBAL_INIT_THREAD_LOCAL_BUFFER)
* _res_key libc * _res_key libc
*/
#define LIBC_PTHREAD_KEY_RESERVED_COUNT 12
#if defined(USE_JEMALLOC)
/* Following are current pthread keys used internally by jemalloc:
* je_thread_allocated_tsd jemalloc * je_thread_allocated_tsd jemalloc
* je_arenas_tsd jemalloc * je_arenas_tsd jemalloc
* je_tcache_tsd jemalloc * je_tcache_tsd jemalloc
* je_tcache_enabled_tsd jemalloc * je_tcache_enabled_tsd jemalloc
* je_quarantine_tsd jemalloc * je_quarantine_tsd jemalloc
*
*/ */
#define JEMALLOC_PTHREAD_KEY_RESERVED_COUNT 5
#define LIBC_TLS_RESERVED_SLOTS 12 #define BIONIC_PTHREAD_KEY_RESERVED_COUNT (LIBC_PTHREAD_KEY_RESERVED_COUNT + JEMALLOC_PTHREAD_KEY_RESERVED_COUNT)
#if defined(USE_JEMALLOC)
/* jemalloc uses 5 keys for itself. */
#define BIONIC_TLS_RESERVED_SLOTS (LIBC_TLS_RESERVED_SLOTS + 5)
#else #else
#define BIONIC_TLS_RESERVED_SLOTS LIBC_TLS_RESERVED_SLOTS #define BIONIC_PTHREAD_KEY_RESERVED_COUNT LIBC_PTHREAD_KEY_RESERVED_COUNT
#endif #endif
/* /*
* Maximum number of elements in the TLS array. * Maximum number of pthread keys allocated.
* This includes space for pthread keys and our own internal slots. * This includes pthread keys used internally and externally.
*/ */
#define BIONIC_TLS_SLOTS (PTHREAD_KEYS_MAX + TLS_SLOT_FIRST_USER_SLOT + BIONIC_TLS_RESERVED_SLOTS) #define BIONIC_PTHREAD_KEY_COUNT (BIONIC_PTHREAD_KEY_RESERVED_COUNT + PTHREAD_KEYS_MAX)
__END_DECLS __END_DECLS