diff --git a/libc/private/bionic_atomic_arm.h b/libc/private/bionic_atomic_arm.h index e94129ca6..3957d465a 100644 --- a/libc/private/bionic_atomic_arm.h +++ b/libc/private/bionic_atomic_arm.h @@ -16,172 +16,64 @@ #ifndef BIONIC_ATOMIC_ARM_H #define BIONIC_ATOMIC_ARM_H -#include - -/* Some of the harware instructions used below are not available in Thumb-1 - * mode (they are if you build in ARM or Thumb-2 mode though). To solve this - * problem, we're going to use the same technique than libatomics_ops, - * which is to temporarily switch to ARM, do the operation, then switch - * back to Thumb-1. - * - * This results in two 'bx' jumps, just like a normal function call, but - * everything is kept inlined, avoids loading or computing the function's - * address, and prevents a little I-cache trashing too. - * - * However, it is highly recommended to avoid compiling any C library source - * file that use these functions in Thumb-1 mode. - * - * Define three helper macros to implement this: - */ -#if defined(__thumb__) && !defined(__thumb2__) -# define __ATOMIC_SWITCH_TO_ARM \ - "adr r3, 5f\n" \ - "bx r3\n" \ - ".align\n" \ - ".arm\n" \ - "5:\n" -/* note: the leading \n below is intentional */ -# define __ATOMIC_SWITCH_TO_THUMB \ - "\n" \ - "adr r3, 6f\n" \ - "bx r3\n" \ - ".thumb" \ - "6:\n" - -# define __ATOMIC_CLOBBERS "r3" /* list of clobbered registers */ - -/* Warn the user that ARM mode should really be preferred! */ -# warning Rebuilding this source file in ARM mode is highly recommended for performance!! - -#else -# define __ATOMIC_SWITCH_TO_ARM /* nothing */ -# define __ATOMIC_SWITCH_TO_THUMB /* nothing */ -# define __ATOMIC_CLOBBERS /* nothing */ -#endif - - -/* Define a full memory barrier, this is only needed if we build the - * platform for a multi-core device. For the record, using a 'dmb' - * instruction on a Nexus One device can take up to 180 ns even if - * it is completely un-necessary on this device. - * - * NOTE: This is where the platform and NDK headers atomic headers are - * going to diverge. With the NDK, we don't know if the generated - * code is going to run on a single or multi-core device, so we - * need to be cautious. - * - * I.e. on single-core devices, the helper immediately returns, - * on multi-core devices, it uses "dmb" or any other means to - * perform a full-memory barrier. - * - * There are three cases to consider for the platform: - * - * - multi-core ARMv7-A => use the 'dmb' hardware instruction - * - multi-core ARMv6 => use the coprocessor - * - single core ARMv6+ => do not use any hardware barrier - */ +__ATOMIC_INLINE__ void __bionic_memory_barrier(void) { #if defined(ANDROID_SMP) && ANDROID_SMP == 1 - -/* For ARMv7-A, we can use the 'dmb' instruction directly */ -__ATOMIC_INLINE__ void __bionic_memory_barrier(void) { - /* Note: we always build in ARM or Thumb-2 on ARMv7-A, so don't - * bother with __ATOMIC_SWITCH_TO_ARM */ - __asm__ __volatile__ ( "dmb" : : : "memory" ); + __asm__ __volatile__ ( "dmb" : : : "memory" ); +#else + /* A simple compiler barrier. */ + __asm__ __volatile__ ( "" : : : "memory" ); +#endif } -#else /* !ANDROID_SMP */ - -__ATOMIC_INLINE__ void __bionic_memory_barrier(void) { - /* A simple compiler barrier */ - __asm__ __volatile__ ( "" : : : "memory" ); -} - -#endif /* !ANDROID_SMP */ - /* Compare-and-swap, without any explicit barriers. Note that this functions * returns 0 on success, and 1 on failure. The opposite convention is typically * used on other platforms. */ -__ATOMIC_INLINE__ int -__bionic_cmpxchg(int32_t old_value, int32_t new_value, volatile int32_t* ptr) -{ - int32_t prev, status; - do { - __asm__ __volatile__ ( - __ATOMIC_SWITCH_TO_ARM - "ldrex %0, [%3]\n" - "mov %1, #0\n" - "teq %0, %4\n" +__ATOMIC_INLINE__ int __bionic_cmpxchg(int32_t old_value, int32_t new_value, volatile int32_t* ptr) { + int32_t prev, status; + do { + __asm__ __volatile__ ( + "ldrex %0, [%3]\n" + "mov %1, #0\n" + "teq %0, %4\n" #ifdef __thumb2__ - "it eq\n" + "it eq\n" #endif - "strexeq %1, %5, [%3]" - __ATOMIC_SWITCH_TO_THUMB - : "=&r" (prev), "=&r" (status), "+m"(*ptr) - : "r" (ptr), "Ir" (old_value), "r" (new_value) - : __ATOMIC_CLOBBERS "cc"); - } while (__builtin_expect(status != 0, 0)); - return prev != old_value; + "strexeq %1, %5, [%3]" + : "=&r" (prev), "=&r" (status), "+m"(*ptr) + : "r" (ptr), "Ir" (old_value), "r" (new_value) + : "cc"); + } while (__builtin_expect(status != 0, 0)); + return prev != old_value; } -/* Swap operation, without any explicit barriers. */ -__ATOMIC_INLINE__ int32_t -__bionic_swap(int32_t new_value, volatile int32_t* ptr) -{ - int32_t prev, status; - do { - __asm__ __volatile__ ( - __ATOMIC_SWITCH_TO_ARM - "ldrex %0, [%3]\n" - "strex %1, %4, [%3]" - __ATOMIC_SWITCH_TO_THUMB - : "=&r" (prev), "=&r" (status), "+m" (*ptr) - : "r" (ptr), "r" (new_value) - : __ATOMIC_CLOBBERS "cc"); - } while (__builtin_expect(status != 0, 0)); - return prev; +/* Swap, without any explicit barriers. */ +__ATOMIC_INLINE__ int32_t __bionic_swap(int32_t new_value, volatile int32_t* ptr) { + int32_t prev, status; + do { + __asm__ __volatile__ ( + "ldrex %0, [%3]\n" + "strex %1, %4, [%3]" + : "=&r" (prev), "=&r" (status), "+m" (*ptr) + : "r" (ptr), "r" (new_value) + : "cc"); + } while (__builtin_expect(status != 0, 0)); + return prev; } -/* Atomic increment - without any barriers - * This returns the old value - */ -__ATOMIC_INLINE__ int32_t -__bionic_atomic_inc(volatile int32_t* ptr) -{ - int32_t prev, tmp, status; - do { - __asm__ __volatile__ ( - __ATOMIC_SWITCH_TO_ARM - "ldrex %0, [%4]\n" - "add %1, %0, #1\n" - "strex %2, %1, [%4]" - __ATOMIC_SWITCH_TO_THUMB - : "=&r" (prev), "=&r" (tmp), "=&r" (status), "+m"(*ptr) - : "r" (ptr) - : __ATOMIC_CLOBBERS "cc"); - } while (__builtin_expect(status != 0, 0)); - return prev; -} - -/* Atomic decrement - without any barriers - * This returns the old value. - */ -__ATOMIC_INLINE__ int32_t -__bionic_atomic_dec(volatile int32_t* ptr) -{ - int32_t prev, tmp, status; - do { - __asm__ __volatile__ ( - __ATOMIC_SWITCH_TO_ARM - "ldrex %0, [%4]\n" - "sub %1, %0, #1\n" - "strex %2, %1, [%4]" - __ATOMIC_SWITCH_TO_THUMB - : "=&r" (prev), "=&r" (tmp), "=&r" (status), "+m"(*ptr) - : "r" (ptr) - : __ATOMIC_CLOBBERS "cc"); - } while (__builtin_expect(status != 0, 0)); - return prev; +/* Atomic decrement, without explicit barriers. */ +__ATOMIC_INLINE__ int32_t __bionic_atomic_dec(volatile int32_t* ptr) { + int32_t prev, tmp, status; + do { + __asm__ __volatile__ ( + "ldrex %0, [%4]\n" + "sub %1, %0, #1\n" + "strex %2, %1, [%4]" + : "=&r" (prev), "=&r" (tmp), "=&r" (status), "+m"(*ptr) + : "r" (ptr) + : "cc"); + } while (__builtin_expect(status != 0, 0)); + return prev; } #endif /* SYS_ATOMICS_ARM_H */ diff --git a/libc/private/bionic_atomic_gcc_builtin.h b/libc/private/bionic_atomic_gcc_builtin.h index 2919f7fef..9e5e5aacd 100644 --- a/libc/private/bionic_atomic_gcc_builtin.h +++ b/libc/private/bionic_atomic_gcc_builtin.h @@ -16,46 +16,35 @@ #ifndef BIONIC_ATOMIC_GCC_BUILTIN_H #define BIONIC_ATOMIC_GCC_BUILTIN_H -/* This header file is used by default if we don't have optimized atomic +/* + * This header file is used by default if we don't have optimized atomic * routines for a given platform. See bionic_atomic_arm.h and * bionic_atomic_x86.h for examples. + * + * Note that the GCC builtins include barriers that aren't present in + * the architecture-specific assembler versions. */ -__ATOMIC_INLINE__ void -__bionic_memory_barrier(void) -{ - __sync_synchronize(); +__ATOMIC_INLINE__ void __bionic_memory_barrier(void) { + __sync_synchronize(); } -__ATOMIC_INLINE__ int -__bionic_cmpxchg(int32_t old_value, int32_t new_value, volatile int32_t* ptr) -{ - /* We must return 0 on success */ - return __sync_val_compare_and_swap(ptr, old_value, new_value) != old_value; +__ATOMIC_INLINE__ int __bionic_cmpxchg(int32_t old_value, int32_t new_value, volatile int32_t* ptr) { + /* We must return 0 on success. */ + return __sync_val_compare_and_swap(ptr, old_value, new_value) != old_value; } -__ATOMIC_INLINE__ int32_t -__bionic_swap(int32_t new_value, volatile int32_t* ptr) -{ - int32_t old_value; - do { - old_value = *ptr; - } while (__sync_val_compare_and_swap(ptr, old_value, new_value) != old_value); - return old_value; +__ATOMIC_INLINE__ int32_t __bionic_swap(int32_t new_value, volatile int32_t* ptr) { + int32_t old_value; + do { + old_value = *ptr; + } while (__sync_val_compare_and_swap(ptr, old_value, new_value) != old_value); + return old_value; } -__ATOMIC_INLINE__ int32_t -__bionic_atomic_inc(volatile int32_t* ptr) -{ - /* We must return the old value */ - return __sync_fetch_and_add(ptr, 1); -} - -__ATOMIC_INLINE__ int32_t -__bionic_atomic_dec(volatile int32_t* ptr) -{ - /* We must return the old value */ - return __sync_fetch_and_add(ptr, -1); +__ATOMIC_INLINE__ int32_t __bionic_atomic_dec(volatile int32_t* ptr) { + /* We must return the old value. */ + return __sync_fetch_and_add(ptr, -1); } #endif /* BIONIC_ATOMIC_GCC_BUILTIN_H */ diff --git a/libc/private/bionic_atomic_inline.h b/libc/private/bionic_atomic_inline.h index 6819af6b3..4936fe15d 100644 --- a/libc/private/bionic_atomic_inline.h +++ b/libc/private/bionic_atomic_inline.h @@ -23,11 +23,6 @@ * memory barrier needs to be issued inline rather than as a function * call. * - * Most code should not use these. - * - * Anything that does include this file must set ANDROID_SMP to either - * 0 or 1, indicating compilation for UP or SMP, respectively. - * * Macros defined in this header: * * void ANDROID_MEMBAR_FULL(void) diff --git a/libc/private/bionic_atomic_mips.h b/libc/private/bionic_atomic_mips.h index 28fe88dd0..5e081161d 100644 --- a/libc/private/bionic_atomic_mips.h +++ b/libc/private/bionic_atomic_mips.h @@ -19,84 +19,58 @@ /* Define a full memory barrier, this is only needed if we build the * platform for a multi-core device. */ + +__ATOMIC_INLINE__ void __bionic_memory_barrier() { #if defined(ANDROID_SMP) && ANDROID_SMP == 1 -__ATOMIC_INLINE__ void -__bionic_memory_barrier() -{ - __asm__ __volatile__ ( "sync" : : : "memory" ); -} + __asm__ __volatile__ ( "sync" : : : "memory" ); #else -__ATOMIC_INLINE__ void -__bionic_memory_barrier() -{ - /* A simple compiler barrier */ - __asm__ __volatile__ ( "" : : : "memory" ); -} + /* A simple compiler barrier. */ + __asm__ __volatile__ ( "" : : : "memory" ); #endif +} /* Compare-and-swap, without any explicit barriers. Note that this function * returns 0 on success, and 1 on failure. The opposite convention is typically * used on other platforms. */ -__ATOMIC_INLINE__ int -__bionic_cmpxchg(int32_t old_value, int32_t new_value, volatile int32_t* ptr) -{ - int32_t prev, status; - __asm__ __volatile__ ("1: move %[status], %[new_value] \n" - " ll %[prev], 0(%[ptr]) \n" - " bne %[old_value], %[prev], 2f \n" - " sc %[status], 0(%[ptr]) \n" - " beqz %[status], 1b \n" - "2: \n" - : [prev]"=&r"(prev), [status]"=&r"(status), "+m"(*ptr) - : [new_value]"r"(new_value), [old_value]"r"(old_value), [ptr]"r"(ptr) - : "memory"); - return prev != old_value; +__ATOMIC_INLINE__ int __bionic_cmpxchg(int32_t old_value, int32_t new_value, volatile int32_t* ptr) { + int32_t prev, status; + __asm__ __volatile__ ("1: move %[status], %[new_value] \n" + " ll %[prev], 0(%[ptr]) \n" + " bne %[old_value], %[prev], 2f \n" + " sc %[status], 0(%[ptr]) \n" + " beqz %[status], 1b \n" + "2: \n" + : [prev]"=&r"(prev), [status]"=&r"(status), "+m"(*ptr) + : [new_value]"r"(new_value), [old_value]"r"(old_value), [ptr]"r"(ptr) + : "memory"); + return prev != old_value; } - -/* Swap, without any explicit barriers */ -__ATOMIC_INLINE__ int32_t -__bionic_swap(int32_t new_value, volatile int32_t *ptr) -{ - int32_t prev, status; - __asm__ __volatile__ ("1: move %[status], %[new_value] \n" - " ll %[prev], 0(%[ptr]) \n" - " sc %[status], 0(%[ptr]) \n" - " beqz %[status], 1b \n" - : [prev]"=&r"(prev), [status]"=&r"(status), "+m"(*ptr) - : [ptr]"r"(ptr), [new_value]"r"(new_value) - : "memory"); - return prev; +/* Swap, without any explicit barriers. */ +__ATOMIC_INLINE__ int32_t __bionic_swap(int32_t new_value, volatile int32_t* ptr) { + int32_t prev, status; + __asm__ __volatile__ ("1: move %[status], %[new_value] \n" + " ll %[prev], 0(%[ptr]) \n" + " sc %[status], 0(%[ptr]) \n" + " beqz %[status], 1b \n" + : [prev]"=&r"(prev), [status]"=&r"(status), "+m"(*ptr) + : [ptr]"r"(ptr), [new_value]"r"(new_value) + : "memory"); + return prev; } -/* Atomic increment, without explicit barriers */ -__ATOMIC_INLINE__ int32_t -__bionic_atomic_inc(volatile int32_t *ptr) -{ - int32_t prev, status; - __asm__ __volatile__ ("1: ll %[prev], 0(%[ptr]) \n" - " addiu %[status], %[prev], 1 \n" - " sc %[status], 0(%[ptr]) \n" - " beqz %[status], 1b \n" - : [prev]"=&r" (prev), [status]"=&r"(status), "+m" (*ptr) - : [ptr]"r"(ptr) - : "memory"); - return prev; +/* Atomic decrement, without explicit barriers. */ +__ATOMIC_INLINE__ int32_t __bionic_atomic_dec(volatile int32_t* ptr) { + int32_t prev, status; + __asm__ __volatile__ ("1: ll %[prev], 0(%[ptr]) \n" + " addiu %[status], %[prev], -1 \n" + " sc %[status], 0(%[ptr]) \n" + " beqz %[status], 1b \n" + : [prev]"=&r" (prev), [status]"=&r"(status), "+m" (*ptr) + : [ptr]"r"(ptr) + : "memory"); + return prev; } -/* Atomic decrement, without explicit barriers */ -__ATOMIC_INLINE__ int32_t -__bionic_atomic_dec(volatile int32_t *ptr) -{ - int32_t prev, status; - __asm__ __volatile__ ("1: ll %[prev], 0(%[ptr]) \n" - " addiu %[status], %[prev], -1 \n" - " sc %[status], 0(%[ptr]) \n" - " beqz %[status], 1b \n" - : [prev]"=&r" (prev), [status]"=&r"(status), "+m" (*ptr) - : [ptr]"r"(ptr) - : "memory"); - return prev; -} #endif /* BIONIC_ATOMIC_MIPS_H */ diff --git a/libc/private/bionic_atomic_x86.h b/libc/private/bionic_atomic_x86.h index aca0c4b56..89639c8b2 100644 --- a/libc/private/bionic_atomic_x86.h +++ b/libc/private/bionic_atomic_x86.h @@ -19,28 +19,20 @@ /* Define a full memory barrier, this is only needed if we build the * platform for a multi-core device. */ +__ATOMIC_INLINE__ void __bionic_memory_barrier() { #if defined(ANDROID_SMP) && ANDROID_SMP == 1 -__ATOMIC_INLINE__ void -__bionic_memory_barrier() -{ - __asm__ __volatile__ ( "mfence" : : : "memory" ); -} + __asm__ __volatile__ ( "mfence" : : : "memory" ); #else -__ATOMIC_INLINE__ void -__bionic_memory_barrier() -{ - /* A simple compiler barrier */ - __asm__ __volatile__ ( "" : : : "memory" ); -} + /* A simple compiler barrier. */ + __asm__ __volatile__ ( "" : : : "memory" ); #endif +} /* Compare-and-swap, without any explicit barriers. Note that this function * returns 0 on success, and 1 on failure. The opposite convention is typically * used on other platforms. */ -__ATOMIC_INLINE__ int -__bionic_cmpxchg(int32_t old_value, int32_t new_value, volatile int32_t* ptr) -{ +__ATOMIC_INLINE__ int __bionic_cmpxchg(int32_t old_value, int32_t new_value, volatile int32_t* ptr) { int32_t prev; __asm__ __volatile__ ("lock; cmpxchgl %1, %2" : "=a" (prev) @@ -49,40 +41,23 @@ __bionic_cmpxchg(int32_t old_value, int32_t new_value, volatile int32_t* ptr) return prev != old_value; } - -/* Swap, without any explicit barriers */ -__ATOMIC_INLINE__ int32_t -__bionic_swap(int32_t new_value, volatile int32_t *ptr) -{ - __asm__ __volatile__ ("xchgl %1, %0" - : "=r" (new_value) - : "m" (*ptr), "0" (new_value) - : "memory"); - return new_value; +/* Swap, without any explicit barriers. */ +__ATOMIC_INLINE__ int32_t __bionic_swap(int32_t new_value, volatile int32_t *ptr) { + __asm__ __volatile__ ("xchgl %1, %0" + : "=r" (new_value) + : "m" (*ptr), "0" (new_value) + : "memory"); + return new_value; } -/* Atomic increment, without explicit barriers */ -__ATOMIC_INLINE__ int32_t -__bionic_atomic_inc(volatile int32_t *ptr) -{ - int increment = 1; - __asm__ __volatile__ ("lock; xaddl %0, %1" - : "+r" (increment), "+m" (*ptr) - : : "memory"); - /* increment now holds the old value of *ptr */ - return increment; -} - -/* Atomic decrement, without explicit barriers */ -__ATOMIC_INLINE__ int32_t -__bionic_atomic_dec(volatile int32_t *ptr) -{ - int increment = -1; - __asm__ __volatile__ ("lock; xaddl %0, %1" - : "+r" (increment), "+m" (*ptr) - : : "memory"); - /* increment now holds the old value of *ptr */ - return increment; +/* Atomic decrement, without explicit barriers. */ +__ATOMIC_INLINE__ int32_t __bionic_atomic_dec(volatile int32_t* ptr) { + int increment = -1; + __asm__ __volatile__ ("lock; xaddl %0, %1" + : "+r" (increment), "+m" (*ptr) + : : "memory"); + /* increment now holds the old value of *ptr */ + return increment; } #endif /* BIONIC_ATOMIC_X86_H */