generic-library/bionic
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge "libc: provide atomic operations will full barriers for NDK apps."

Browse Source
This commit is contained in:
David 'Digit' Turner 2011-11-22 02:10:06 -08:00 committed by Android (Google) Code Review
commit de44d0b2bd
7 changed files with 244 additions and 307 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
libc/Android.mk
View File

@ -346,10 +346,11 @@ libc_common_src_files += \
arch-arm/bionic/__get_sp.S \
arch-arm/bionic/_exit_with_stack_teardown.S \
arch-arm/bionic/_setjmp.S \
arch-arm/bionic/atomics_arm.S \
arch-arm/bionic/atomics_arm.c \
arch-arm/bionic/clone.S \
arch-arm/bionic/eabi.c \
arch-arm/bionic/ffs.S \
arch-arm/bionic/futex_arm.S \
arch-arm/bionic/kill.S \
arch-arm/bionic/libgcc_compat.c \
arch-arm/bionic/tkill.S \
@ -395,9 +396,9 @@ libc_common_src_files += \
arch-x86/bionic/__get_sp.S \
arch-x86/bionic/__get_tls.c \
arch-x86/bionic/__set_tls.c \
arch-x86/bionic/atomics_x86.S \
arch-x86/bionic/clone.S \
arch-x86/bionic/_exit_with_stack_teardown.S \
arch-x86/bionic/futex_x86.S \
arch-x86/bionic/setjmp.S \
arch-x86/bionic/_setjmp.S \
arch-x86/bionic/sigsetjmp.S \

236
libc/arch-arm/bionic/atomics_arm.S
View File

@ -1,236 +0,0 @@
/*
* Copyright (C) 2008 The Android Open Source Project
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/linux-syscalls.h>
#include <machine/asm.h>
#include <machine/cpu-features.h>
#define FUTEX_WAIT 0
#define FUTEX_WAKE 1
#if defined(__ARM_HAVE_LDREX_STREX)
/*
* ===========================================================================
* ARMv6+ implementation
* ===========================================================================
*/
/* r0(addr) -> r0(old) */
ENTRY(__atomic_dec)
mov r1, r0 @ copy addr so we don't clobber it
1: ldrex r0, [r1] @ load current value into r0
sub r2, r0, #1 @ generate new value into r2
strex r3, r2, [r1] @ try to store new value; result in r3
cmp r3, #0 @ success?
bxeq lr @ yes, return
b 1b @ no, retry
END(__atomic_dec)
/* r0(addr) -> r0(old) */
ENTRY(__atomic_inc)
mov r1, r0
1: ldrex r0, [r1]
add r2, r0, #1
strex r3, r2, [r1]
cmp r3, #0
bxeq lr
b 1b
END(__atomic_inc)
/* r0(old) r1(new) r2(addr) -> r0(zero_if_succeeded) */
ENTRY(__atomic_cmpxchg)
1: mov ip, #2 @ ip=2 means "new != old"
ldrex r3, [r2] @ load current value into r3
teq r0, r3 @ new == old?
strexeq ip, r1, [r2] @ yes, try store, set ip to 0 or 1
teq ip, #1 @ strex failure?
beq 1b @ yes, retry
mov r0, ip @ return 0 on success, 2 on failure
bx lr
END(__atomic_cmpxchg)
/* r0(new) r1(addr) -> r0(old) */
ENTRY(__atomic_swap)
1: ldrex r2, [r1]
strex r3, r0, [r1]
teq r3, #0
bne 1b
mov r0, r2
bx lr
END(__atomic_swap)
#else /*not defined __ARM_HAVE_LDREX_STREX*/
/*
* ===========================================================================
* Pre-ARMv6 implementation
* ===========================================================================
*/
/* int __kernel_cmpxchg(int oldval, int newval, int* ptr) */
.equ kernel_cmpxchg, 0xFFFF0FC0
.equ kernel_atomic_base, 0xFFFF0FFF
/* r0(addr) -> r0(old) */
ENTRY(__atomic_dec)
.save {r4, lr}
stmdb sp!, {r4, lr}
mov r2, r0
1: @ atomic_dec
ldr r0, [r2]
mov r3, #kernel_atomic_base
add lr, pc, #4
sub r1, r0, #1
add pc, r3, #(kernel_cmpxchg - kernel_atomic_base)
bcc 1b
add r0, r1, #1
ldmia sp!, {r4, lr}
bx lr
END(__atomic_dec)
/* r0(addr) -> r0(old) */
ENTRY(__atomic_inc)
.save {r4, lr}
stmdb sp!, {r4, lr}
mov r2, r0
1: @ atomic_inc
ldr r0, [r2]
mov r3, #kernel_atomic_base
add lr, pc, #4
add r1, r0, #1
add pc, r3, #(kernel_cmpxchg - kernel_atomic_base)
bcc 1b
sub r0, r1, #1
ldmia sp!, {r4, lr}
bx lr
END(__atomic_inc)
/* r0(old) r1(new) r2(addr) -> r0(zero_if_succeeded) */
ENTRY(__atomic_cmpxchg)
.save {r4, lr}
stmdb sp!, {r4, lr}
mov r4, r0 /* r4 = save oldvalue */
1: @ atomic_cmpxchg
mov r3, #kernel_atomic_base
add lr, pc, #4
mov r0, r4 /* r0 = oldvalue */
add pc, r3, #(kernel_cmpxchg - kernel_atomic_base)
bcs 2f /* swap was made. we're good, return. */
ldr r3, [r2] /* swap not made, see if it's because *ptr!=oldvalue */
cmp r3, r4
beq 1b
2: @ atomic_cmpxchg
ldmia sp!, {r4, lr}
bx lr
END(__atomic_cmpxchg)
/* r0(new) r1(addr) -> r0(old) */
ENTRY(__atomic_swap)
swp r0, r0, [r1]
bx lr
END(__atomic_swap)
#endif /*not defined __ARM_HAVE_LDREX_STREX*/
/* __futex_wait(*ftx, val, *timespec) */
/* __futex_wake(*ftx, counter) */
/* __futex_syscall3(*ftx, op, val) */
/* __futex_syscall4(*ftx, op, val, *timespec) */
.global __futex_wait
.type __futex_wait, %function
.global __futex_wake
.type __futex_wake, %function
.global __futex_syscall3
.type __futex_syscall3, %function
.global __futex_syscall4
.type __futex_syscall4, %function
#if __ARM_EABI__
ENTRY(__futex_syscall3)
stmdb sp!, {r4, r7}
.save {r4, r7}
ldr r7, =__NR_futex
swi #0
ldmia sp!, {r4, r7}
bx lr
END(__futex_syscall3)
ENTRY(__futex_wait)
stmdb sp!, {r4, r7}
.save {r4, r7}
mov r3, r2
mov r2, r1
mov r1, #FUTEX_WAIT
ldr r7, =__NR_futex
swi #0
ldmia sp!, {r4, r7}
bx lr
END(__futex_wait)
ENTRY(__futex_wake)
.save {r4, r7}
stmdb sp!, {r4, r7}
mov r2, r1
mov r1, #FUTEX_WAKE
ldr r7, =__NR_futex
swi #0
ldmia sp!, {r4, r7}
bx lr
END(__futex_wake)
#else
ENTRY(__futex_syscall3)
swi #__NR_futex
bx lr
END(__futex_syscall3)
ENTRY(__futex_wait)
mov r3, r2
mov r2, r1
mov r1, #FUTEX_WAIT
swi #__NR_futex
bx lr
END(__futex_wait)
ENTRY(__futex_wake)
mov r2, r1
mov r1, #FUTEX_WAKE
swi #__NR_futex
bx lr
END(__futex_wake)
#endif
ENTRY(__futex_syscall4)
b __futex_syscall3
END(__futex_syscall4)

87
libc/arch-arm/bionic/atomics_arm.c Normal file
View File

@ -0,0 +1,87 @@
/*
* Copyright (C) 2011 The Android Open Source Project
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/* The purpose of this file is to export a small set of atomic-related
* functions from the C library, to ensure binary ABI compatibility for
* the NDK.
*
* These functions were initially exposed by the NDK through <sys/atomics.h>,
* which was unfortunate because their implementation didn't provide any
* memory barriers at all.
*
* This wasn't a problem for the platform code that used them, because it
* used explicit barrier instructions around them. On the other hand, it means
* that any NDK-generated machine code that linked against them would not
* perform correctly when running on multi-core devices.
*
* To fix this, the platform code was first modified to not use any of these
* functions (everything is now inlined through assembly statements, see
* libc/private/bionic_arm_inline.h and the headers it includes.
*
* The functions here are thus only for the benefit of NDK applications,
* and now includes full memory barriers to prevent any random memory ordering
* issue from cropping.
*
* Note that we also provide an updated <sys/atomics.h> header that defines
* always_inlined versions of the functions that use the GCC builtin
* intrinsics to perform the same thing.
*
* NOTE: There is no need for a similar file for non-ARM platforms.
*/
/* DO NOT INCLUDE <sys/atomics.h> HERE ! */
int
__android_cmpxchg(int old, int _new, volatile int *ptr)
{
/* We must return 0 on success */
return __sync_val_compare_and_swap(ptr, old, _new) != old;
}
int
__atomic_swap(int _new, volatile int *ptr)
{
int prev;
do {
prev = *ptr;
} while (__sync_val_compare_and_swap(ptr, prev, _new) != prev);
return prev;
}
int
__atomic_dec(volatile int *ptr)
{
return __sync_fetch_and_sub (ptr, 1);
}
int
__atomic_inc(volatile int *ptr)
{
return __sync_fetch_and_add (ptr, 1);
}

112
libc/arch-arm/bionic/futex_arm.S Normal file
View File

@ -0,0 +1,112 @@
/*
* Copyright (C) 2008 The Android Open Source Project
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/linux-syscalls.h>
#include <machine/asm.h>
#include <machine/cpu-features.h>
#define FUTEX_WAIT 0
#define FUTEX_WAKE 1
/* __futex_wait(*ftx, val, *timespec) */
/* __futex_wake(*ftx, counter) */
/* __futex_syscall3(*ftx, op, val) */
/* __futex_syscall4(*ftx, op, val, *timespec) */
.global __futex_wait
.type __futex_wait, %function
.global __futex_wake
.type __futex_wake, %function
.global __futex_syscall3
.type __futex_syscall3, %function
.global __futex_syscall4
.type __futex_syscall4, %function
#if __ARM_EABI__
ENTRY(__futex_syscall3)
stmdb sp!, {r4, r7}
.save {r4, r7}
ldr r7, =__NR_futex
swi #0
ldmia sp!, {r4, r7}
bx lr
END(__futex_syscall3)
ENTRY(__futex_wait)
stmdb sp!, {r4, r7}
.save {r4, r7}
mov r3, r2
mov r2, r1
mov r1, #FUTEX_WAIT
ldr r7, =__NR_futex
swi #0
ldmia sp!, {r4, r7}
bx lr
END(__futex_wait)
ENTRY(__futex_wake)
.save {r4, r7}
stmdb sp!, {r4, r7}
mov r2, r1
mov r1, #FUTEX_WAKE
ldr r7, =__NR_futex
swi #0
ldmia sp!, {r4, r7}
bx lr
END(__futex_wake)
#else
ENTRY(__futex_syscall3)
swi #__NR_futex
bx lr
END(__futex_syscall3)
ENTRY(__futex_wait)
mov r3, r2
mov r2, r1
mov r1, #FUTEX_WAIT
swi #__NR_futex
bx lr
END(__futex_wait)
ENTRY(__futex_wake)
mov r2, r1
mov r1, #FUTEX_WAKE
swi #__NR_futex
bx lr
END(__futex_wake)
#endif
ENTRY(__futex_syscall4)
b __futex_syscall3
END(__futex_syscall4)

0
libc/arch-x86/bionic/atomics_x86.S → libc/arch-x86/bionic/futex_x86.S
View File

65
libc/arch-x86/include/sys/atomics.h
View File

@ -1,65 +0,0 @@
/*
* Copyright (C) 2011 The Android Open Source Project
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _SYS_ATOMICS_H
#define _SYS_ATOMICS_H
#include <sys/cdefs.h>
#include <sys/time.h>
__BEGIN_DECLS
static inline __attribute__((always_inline)) int
__atomic_cmpxchg(int old, int _new, volatile int *ptr)
{
return !__sync_bool_compare_and_swap (ptr, old, _new);
}
static inline __attribute__((always_inline)) int
__atomic_swap(int _new, volatile int *ptr)
{
return __sync_lock_test_and_set(ptr, _new);
}
static inline __attribute__((always_inline)) int
__atomic_dec(volatile int *ptr)
{
return __sync_fetch_and_sub (ptr, 1);
}
static inline __attribute__((always_inline)) int
__atomic_inc(volatile int *ptr)
{
return __sync_fetch_and_add (ptr, 1);
}
int __futex_wait(volatile void *ftx, int val, const struct timespec *timeout);
int __futex_wake(volatile void *ftx, int count);
__END_DECLS
#endif /* _SYS_ATOMICS_H */

46
libc/include/sys/atomics.h
View File

@ -33,10 +33,48 @@
__BEGIN_DECLS
extern int __atomic_cmpxchg(int old, int _new, volatile int *ptr);
extern int __atomic_swap(int _new, volatile int *ptr);
extern int __atomic_dec(volatile int *ptr);
extern int __atomic_inc(volatile int *ptr);
/* Note: atomic operations that were exported by the C library didn't
* provide any memory barriers, which created potential issues on
* multi-core devices. We now define them as inlined calls to
* GCC sync builtins, which always provide a full barrier.
*
* NOTE: The C library still exports atomic functions by the same
* name to ensure ABI stability for existing NDK machine code.
*
* If you are an NDK developer, we encourage you to rebuild your
* unmodified sources against this header as soon as possible.
*/
#define __ATOMIC_INLINE__ static __inline__ __attribute__((always_inline))
__ATOMIC_INLINE__ int
__atomic_cmpxchg(int old, int _new, volatile int *ptr)
{
/* We must return 0 on success */
return __sync_val_compare_and_swap(ptr, old, _new) != old;
}
__ATOMIC_INLINE__ int
__atomic_swap(int _new, volatile int *ptr)
{
int prev;
do {
prev = *ptr;
} while (__sync_val_compare_and_swap(ptr, prev, _new) != prev);
return prev;
}
__ATOMIC_INLINE__ int
__atomic_dec(volatile int *ptr)
{
return __sync_fetch_and_sub (ptr, 1);
}
__ATOMIC_INLINE__ int
__atomic_inc(volatile int *ptr)
{
return __sync_fetch_and_add (ptr, 1);
}
int __futex_wait(volatile void *ftx, int val, const struct timespec *timeout);
int __futex_wake(volatile void *ftx, int count);