diff --git a/libc/Android.mk b/libc/Android.mk index a89fe230b..2849591e4 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -564,6 +564,7 @@ libc_upstream_openbsd_ndk_src_files := \ libc_pthread_src_files := \ bionic/pthread_atfork.cpp \ bionic/pthread_attr.cpp \ + bionic/pthread_barrier.cpp \ bionic/pthread_cond.cpp \ bionic/pthread_create.cpp \ bionic/pthread_detach.cpp \ diff --git a/libc/bionic/pthread_barrier.cpp b/libc/bionic/pthread_barrier.cpp new file mode 100644 index 000000000..3227dafa9 --- /dev/null +++ b/libc/bionic/pthread_barrier.cpp @@ -0,0 +1,183 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include + +#include "private/bionic_futex.h" + +int pthread_barrierattr_init(pthread_barrierattr_t* attr) { + *attr = 0; + return 0; +} + +int pthread_barrierattr_destroy(pthread_barrierattr_t* attr) { + *attr = 0; + return 0; +} + +int pthread_barrierattr_getpshared(pthread_barrierattr_t* attr, int* pshared) { + *pshared = (*attr & 1) ? PTHREAD_PROCESS_SHARED : PTHREAD_PROCESS_PRIVATE; + return 0; +} + +int pthread_barrierattr_setpshared(pthread_barrierattr_t* attr, int pshared) { + if (pshared == PTHREAD_PROCESS_SHARED) { + *attr |= 1; + } else { + *attr &= ~1; + } + return 0; +} + +enum BarrierState { + WAIT, + RELEASE, +}; + +struct pthread_barrier_internal_t { + // One barrier can be used for unlimited number of cycles. In each cycle, [init_count] + // threads must call pthread_barrier_wait() before any of them successfully return from + // the call. It is undefined behavior if there are more than [init_count] threads call + // pthread_barrier_wait() in one cycle. + uint32_t init_count; + // Barrier state. It is WAIT if waiting for more threads to enter the barrier in this cycle, + // otherwise threads are leaving the barrier. + _Atomic(BarrierState) state; + // Number of threads having entered but not left the barrier in this cycle. + atomic_uint wait_count; + // Whether the barrier is shared across processes. + bool pshared; + uint32_t __reserved[4]; +}; + +static_assert(sizeof(pthread_barrier_t) == sizeof(pthread_barrier_internal_t), + "pthread_barrier_t should actually be pthread_barrier_internal_t in implementation." + ); + +static_assert(alignof(pthread_barrier_t) >= 4, + "pthread_barrier_t should fulfill the alignment of pthread_barrier_internal_t."); + +static inline pthread_barrier_internal_t* __get_internal_barrier(pthread_barrier_t* barrier) { + return reinterpret_cast(barrier); +} + +int pthread_barrier_init(pthread_barrier_t* barrier_interface, const pthread_barrierattr_t* attr, + unsigned count) { + pthread_barrier_internal_t* barrier = __get_internal_barrier(barrier_interface); + if (count == 0) { + return EINVAL; + } + barrier->init_count = count; + atomic_init(&barrier->state, WAIT); + atomic_init(&barrier->wait_count, 0); + barrier->pshared = false; + if (attr != nullptr && (*attr & 1)) { + barrier->pshared = true; + } + return 0; +} + +// According to POSIX standard, pthread_barrier_wait() synchronizes memory between participating +// threads. It means all memory operations made by participating threads before calling +// pthread_barrier_wait() can be seen by all participating threads after the function call. +// We establish this by making a happens-before relation between all threads entering the barrier +// with the last thread entering the barrier, and a happens-before relation between the last +// thread entering the barrier with all threads leaving the barrier. +int pthread_barrier_wait(pthread_barrier_t* barrier_interface) { + pthread_barrier_internal_t* barrier = __get_internal_barrier(barrier_interface); + + // Wait until all threads for the previous cycle have left the barrier. This is needed + // as a participating thread can call pthread_barrier_wait() again before other + // threads have left the barrier. Use acquire operation here to synchronize with + // the last thread leaving the previous cycle, so we can read correct wait_count below. + while(atomic_load_explicit(&barrier->state, memory_order_acquire) == RELEASE) { + __futex_wait_ex(&barrier->state, barrier->pshared, RELEASE, nullptr); + } + + uint32_t prev_wait_count = atomic_load_explicit(&barrier->wait_count, memory_order_relaxed); + while (true) { + // It happens when there are more than [init_count] threads trying to enter the barrier + // at one cycle. We read the POSIX standard as disallowing this, since additional arriving + // threads are not synchronized with respect to the barrier reset. We also don't know of + // any reasonable cases in which this would be intentional. + if (prev_wait_count >= barrier->init_count) { + return EINVAL; + } + // Use memory_order_acq_rel operation here to synchronize between all threads entering + // the barrier with the last thread entering the barrier. + if (atomic_compare_exchange_weak_explicit(&barrier->wait_count, &prev_wait_count, + prev_wait_count + 1u, memory_order_acq_rel, + memory_order_relaxed)) { + break; + } + } + + int result = 0; + if (prev_wait_count + 1 == barrier->init_count) { + result = PTHREAD_BARRIER_SERIAL_THREAD; + if (prev_wait_count != 0) { + // Use release operation here to synchronize between the last thread entering the + // barrier with all threads leaving the barrier. + atomic_store_explicit(&barrier->state, RELEASE, memory_order_release); + __futex_wake_ex(&barrier->state, barrier->pshared, prev_wait_count); + } + } else { + // Use acquire operation here to synchronize between the last thread entering the + // barrier with all threads leaving the barrier. + while (atomic_load_explicit(&barrier->state, memory_order_acquire) == WAIT) { + __futex_wait_ex(&barrier->state, barrier->pshared, WAIT, nullptr); + } + } + // Use release operation here to make it not reordered with previous operations. + if (atomic_fetch_sub_explicit(&barrier->wait_count, 1, memory_order_release) == 1) { + // Use release operation here to synchronize with threads entering the barrier for + // the next cycle, or the thread calling pthread_barrier_destroy(). + atomic_store_explicit(&barrier->state, WAIT, memory_order_release); + __futex_wake_ex(&barrier->state, barrier->pshared, barrier->init_count); + } + return result; +} + +int pthread_barrier_destroy(pthread_barrier_t* barrier_interface) { + pthread_barrier_internal_t* barrier = __get_internal_barrier(barrier_interface); + if (barrier->init_count == 0) { + return EINVAL; + } + // Use acquire operation here to synchronize with the last thread leaving the barrier. + // So we can read correct wait_count below. + while (atomic_load_explicit(&barrier->state, memory_order_acquire) == RELEASE) { + __futex_wait_ex(&barrier->state, barrier->pshared, RELEASE, nullptr); + } + if (atomic_load_explicit(&barrier->wait_count, memory_order_relaxed) != 0) { + return EBUSY; + } + barrier->init_count = 0; + return 0; +} diff --git a/libc/include/pthread.h b/libc/include/pthread.h index 260ae5b87..48b2bca2d 100644 --- a/libc/include/pthread.h +++ b/libc/include/pthread.h @@ -96,6 +96,18 @@ typedef int pthread_once_t; #define PTHREAD_ONCE_INIT 0 +typedef struct { +#if defined(__LP64__) + int64_t __private[4]; +#else + int32_t __private[8]; +#endif +} pthread_barrier_t; + +typedef int pthread_barrierattr_t; + +#define PTHREAD_BARRIER_SERIAL_THREAD -1 + #if defined(__LP64__) #define PTHREAD_STACK_MIN (4 * PAGE_SIZE) #else @@ -130,7 +142,7 @@ int pthread_attr_setschedparam(pthread_attr_t*, const struct sched_param*) __non int pthread_attr_setschedpolicy(pthread_attr_t*, int) __nonnull((1)); int pthread_attr_setscope(pthread_attr_t*, int) __nonnull((1)); int pthread_attr_setstack(pthread_attr_t*, void*, size_t) __nonnull((1)); -int pthread_attr_setstacksize(pthread_attr_t*, size_t stack_size) __nonnull((1)); +int pthread_attr_setstacksize(pthread_attr_t*, size_t) __nonnull((1)); int pthread_condattr_destroy(pthread_condattr_t*) __nonnull((1)); int pthread_condattr_getclock(const pthread_condattr_t*, clockid_t*) __nonnull((1, 2)); @@ -205,9 +217,18 @@ int pthread_rwlock_timedrdlock(pthread_rwlock_t*, const struct timespec*) __nonn int pthread_rwlock_timedwrlock(pthread_rwlock_t*, const struct timespec*) __nonnull((1, 2)); int pthread_rwlock_tryrdlock(pthread_rwlock_t*) __nonnull((1)); int pthread_rwlock_trywrlock(pthread_rwlock_t*) __nonnull((1)); -int pthread_rwlock_unlock(pthread_rwlock_t *rwlock) __nonnull((1)); +int pthread_rwlock_unlock(pthread_rwlock_t *) __nonnull((1)); int pthread_rwlock_wrlock(pthread_rwlock_t*) __nonnull((1)); +int pthread_barrierattr_init(pthread_barrierattr_t* attr) __nonnull((1)); +int pthread_barrierattr_destroy(pthread_barrierattr_t* attr) __nonnull((1)); +int pthread_barrierattr_getpshared(pthread_barrierattr_t* attr, int* pshared) __nonnull((1, 2)); +int pthread_barrierattr_setpshared(pthread_barrierattr_t* attr, int pshared) __nonnull((1)); + +int pthread_barrier_init(pthread_barrier_t*, const pthread_barrierattr_t*, unsigned) __nonnull((1)); +int pthread_barrier_destroy(pthread_barrier_t*) __nonnull((1)); +int pthread_barrier_wait(pthread_barrier_t*) __nonnull((1)); + pthread_t pthread_self(void) __pure2; int pthread_setname_np(pthread_t, const char*) __nonnull((2)); diff --git a/libc/libc.arm.map b/libc/libc.arm.map index b632d3b15..f7805fed6 100644 --- a/libc/libc.arm.map +++ b/libc/libc.arm.map @@ -1316,6 +1316,13 @@ LIBC_N { preadv; preadv64; prlimit; # arm mips x86 + pthread_barrierattr_destroy; + pthread_barrierattr_getpshared; + pthread_barrierattr_init; + pthread_barrierattr_setpshared; + pthread_barrier_destroy; + pthread_barrier_init; + pthread_barrier_wait; pwritev; pwritev64; scandirat; diff --git a/libc/libc.arm64.map b/libc/libc.arm64.map index 38023e5bc..d5fe2625b 100644 --- a/libc/libc.arm64.map +++ b/libc/libc.arm64.map @@ -1161,6 +1161,13 @@ LIBC_N { getgrnam_r; preadv; preadv64; + pthread_barrierattr_destroy; + pthread_barrierattr_getpshared; + pthread_barrierattr_init; + pthread_barrierattr_setpshared; + pthread_barrier_destroy; + pthread_barrier_init; + pthread_barrier_wait; pwritev; pwritev64; scandirat; diff --git a/libc/libc.map.txt b/libc/libc.map.txt index c3784561f..738836cd7 100644 --- a/libc/libc.map.txt +++ b/libc/libc.map.txt @@ -1343,6 +1343,13 @@ LIBC_N { preadv; preadv64; prlimit; # arm mips x86 + pthread_barrierattr_destroy; + pthread_barrierattr_getpshared; + pthread_barrierattr_init; + pthread_barrierattr_setpshared; + pthread_barrier_destroy; + pthread_barrier_init; + pthread_barrier_wait; pwritev; pwritev64; scandirat; diff --git a/libc/libc.mips.map b/libc/libc.mips.map index 23123dc0a..2121ba710 100644 --- a/libc/libc.mips.map +++ b/libc/libc.mips.map @@ -1279,6 +1279,13 @@ LIBC_N { preadv; preadv64; prlimit; # arm mips x86 + pthread_barrierattr_destroy; + pthread_barrierattr_getpshared; + pthread_barrierattr_init; + pthread_barrierattr_setpshared; + pthread_barrier_destroy; + pthread_barrier_init; + pthread_barrier_wait; pwritev; pwritev64; scandirat; diff --git a/libc/libc.mips64.map b/libc/libc.mips64.map index 38023e5bc..d5fe2625b 100644 --- a/libc/libc.mips64.map +++ b/libc/libc.mips64.map @@ -1161,6 +1161,13 @@ LIBC_N { getgrnam_r; preadv; preadv64; + pthread_barrierattr_destroy; + pthread_barrierattr_getpshared; + pthread_barrierattr_init; + pthread_barrierattr_setpshared; + pthread_barrier_destroy; + pthread_barrier_init; + pthread_barrier_wait; pwritev; pwritev64; scandirat; diff --git a/libc/libc.x86.map b/libc/libc.x86.map index f13ffb00a..fdee2c647 100644 --- a/libc/libc.x86.map +++ b/libc/libc.x86.map @@ -1277,6 +1277,13 @@ LIBC_N { preadv; preadv64; prlimit; # arm mips x86 + pthread_barrierattr_destroy; + pthread_barrierattr_getpshared; + pthread_barrierattr_init; + pthread_barrierattr_setpshared; + pthread_barrier_destroy; + pthread_barrier_init; + pthread_barrier_wait; pwritev; pwritev64; scandirat; diff --git a/libc/libc.x86_64.map b/libc/libc.x86_64.map index 38023e5bc..d5fe2625b 100644 --- a/libc/libc.x86_64.map +++ b/libc/libc.x86_64.map @@ -1161,6 +1161,13 @@ LIBC_N { getgrnam_r; preadv; preadv64; + pthread_barrierattr_destroy; + pthread_barrierattr_getpshared; + pthread_barrierattr_init; + pthread_barrierattr_setpshared; + pthread_barrier_destroy; + pthread_barrier_init; + pthread_barrier_wait; pwritev; pwritev64; scandirat; diff --git a/tests/pthread_test.cpp b/tests/pthread_test.cpp index e764283b7..e62518ac8 100755 --- a/tests/pthread_test.cpp +++ b/tests/pthread_test.cpp @@ -721,11 +721,11 @@ TEST(pthread, pthread_rwlock_smoke) { ASSERT_EQ(0, pthread_rwlock_destroy(&l)); } -static void WaitUntilThreadSleep(std::atomic& pid) { - while (pid == 0) { +static void WaitUntilThreadSleep(std::atomic& tid) { + while (tid == 0) { usleep(1000); } - std::string filename = android::base::StringPrintf("/proc/%d/stat", pid.load()); + std::string filename = android::base::StringPrintf("/proc/%d/stat", tid.load()); std::regex regex {R"(\s+S\s+)"}; while (true) { @@ -1665,3 +1665,117 @@ TEST(pthread, big_enough_signal_stack_for_64bit_arch) { kill(getpid(), SIGUSR1); ASSERT_TRUE(signal_handler_on_altstack_done); } + +TEST(pthread, pthread_barrierattr_smoke) { + pthread_barrierattr_t attr; + ASSERT_EQ(0, pthread_barrierattr_init(&attr)); + int pshared; + ASSERT_EQ(0, pthread_barrierattr_getpshared(&attr, &pshared)); + ASSERT_EQ(PTHREAD_PROCESS_PRIVATE, pshared); + ASSERT_EQ(0, pthread_barrierattr_setpshared(&attr, PTHREAD_PROCESS_SHARED)); + ASSERT_EQ(0, pthread_barrierattr_getpshared(&attr, &pshared)); + ASSERT_EQ(PTHREAD_PROCESS_SHARED, pshared); + ASSERT_EQ(0, pthread_barrierattr_destroy(&attr)); +} + +struct BarrierTestHelperArg { + std::atomic tid; + pthread_barrier_t* barrier; + size_t iteration_count; +}; + +static void BarrierTestHelper(BarrierTestHelperArg* arg) { + arg->tid = gettid(); + for (size_t i = 0; i < arg->iteration_count; ++i) { + ASSERT_EQ(0, pthread_barrier_wait(arg->barrier)); + } +} + +TEST(pthread, pthread_barrier_smoke) { + const size_t BARRIER_ITERATION_COUNT = 10; + const size_t BARRIER_THREAD_COUNT = 10; + pthread_barrier_t barrier; + ASSERT_EQ(0, pthread_barrier_init(&barrier, nullptr, BARRIER_THREAD_COUNT + 1)); + std::vector threads(BARRIER_THREAD_COUNT); + std::vector args(threads.size()); + for (size_t i = 0; i < threads.size(); ++i) { + args[i].tid = 0; + args[i].barrier = &barrier; + args[i].iteration_count = BARRIER_ITERATION_COUNT; + ASSERT_EQ(0, pthread_create(&threads[i], nullptr, + reinterpret_cast(BarrierTestHelper), &args[i])); + } + for (size_t iteration = 0; iteration < BARRIER_ITERATION_COUNT; ++iteration) { + for (size_t i = 0; i < threads.size(); ++i) { + WaitUntilThreadSleep(args[i].tid); + } + ASSERT_EQ(PTHREAD_BARRIER_SERIAL_THREAD, pthread_barrier_wait(&barrier)); + } + for (size_t i = 0; i < threads.size(); ++i) { + ASSERT_EQ(0, pthread_join(threads[i], nullptr)); + } + ASSERT_EQ(0, pthread_barrier_destroy(&barrier)); +} + +TEST(pthread, pthread_barrier_destroy) { + pthread_barrier_t barrier; + ASSERT_EQ(0, pthread_barrier_init(&barrier, nullptr, 2)); + pthread_t thread; + BarrierTestHelperArg arg; + arg.tid = 0; + arg.barrier = &barrier; + arg.iteration_count = 1; + ASSERT_EQ(0, pthread_create(&thread, nullptr, + reinterpret_cast(BarrierTestHelper), &arg)); + WaitUntilThreadSleep(arg.tid); + ASSERT_EQ(EBUSY, pthread_barrier_destroy(&barrier)); + ASSERT_EQ(PTHREAD_BARRIER_SERIAL_THREAD, pthread_barrier_wait(&barrier)); + // Verify if the barrier can be destroyed directly after pthread_barrier_wait(). + ASSERT_EQ(0, pthread_barrier_destroy(&barrier)); + ASSERT_EQ(0, pthread_join(thread, nullptr)); +#if defined(__BIONIC__) + ASSERT_EQ(EINVAL, pthread_barrier_destroy(&barrier)); +#endif +} + +struct BarrierOrderingTestHelperArg { + pthread_barrier_t* barrier; + size_t* array; + size_t array_length; + size_t id; +}; + +void BarrierOrderingTestHelper(BarrierOrderingTestHelperArg* arg) { + const size_t ITERATION_COUNT = 10000; + for (size_t i = 1; i <= ITERATION_COUNT; ++i) { + arg->array[arg->id] = i; + int ret = pthread_barrier_wait(arg->barrier); + ASSERT_TRUE(ret == 0 || ret == PTHREAD_BARRIER_SERIAL_THREAD); + for (size_t j = 0; j < arg->array_length; ++j) { + ASSERT_EQ(i, arg->array[j]); + } + ret = pthread_barrier_wait(arg->barrier); + ASSERT_TRUE(ret == 0 || ret == PTHREAD_BARRIER_SERIAL_THREAD); + } +} + +TEST(pthread, pthread_barrier_check_ordering) { + const size_t THREAD_COUNT = 4; + pthread_barrier_t barrier; + ASSERT_EQ(0, pthread_barrier_init(&barrier, nullptr, THREAD_COUNT)); + size_t array[THREAD_COUNT]; + std::vector threads(THREAD_COUNT); + std::vector args(THREAD_COUNT); + for (size_t i = 0; i < THREAD_COUNT; ++i) { + args[i].barrier = &barrier; + args[i].array = array; + args[i].array_length = THREAD_COUNT; + args[i].id = i; + ASSERT_EQ(0, pthread_create(&threads[i], nullptr, + reinterpret_cast(BarrierOrderingTestHelper), + &args[i])); + } + for (size_t i = 0; i < THREAD_COUNT; ++i) { + ASSERT_EQ(0, pthread_join(threads[i], nullptr)); + } +}