From b30aff405a220495941f1673b0a5e66c4fa8b84c Mon Sep 17 00:00:00 2001 From: Elliott Hughes Date: Wed, 28 May 2014 19:35:33 +0000 Subject: [PATCH] Revert "Revert "Lose the hand-written futex assembler."" The problem with the original patch was that using syscall(3) means that errno can be set, but pthread_create(3) was abusing the TLS errno slot as a pthread_mutex_t for the thread startup handshake. There was also a mistake in the check for syscall failures --- it should have checked against -1 instead of 0 (not just because that's the default idiom, but also here because futex(2) can legitimately return values > 0). This patch stops abusing the TLS errno slot and adds a pthread_mutex_t to pthread_internal_t instead. (Note that for LP64 sizeof(pthread_mutex_t) > sizeof(uintptr_t), so we could potentially clobber other TLS slots too.) I've also rewritten the LP32 compatibility stubs to directly reuse the code from the .h file. This reverts commit 75c55ff84ebfa686c7ae2cc8ee431c6a33bd46b4. Bug: 15195455 Change-Id: I6ffb13e5cf6a35d8f59f692d94192aae9ab4593d --- libc/SYSCALLS.TXT | 2 - libc/arch-arm/arm.mk | 1 - libc/arch-arm/bionic/futex_arm.S | 38 --------------- libc/arch-arm/syscalls/futex.S | 22 --------- libc/arch-arm64/arm64.mk | 1 - libc/arch-arm64/bionic/futex_arm64.S | 47 ------------------- libc/arch-arm64/syscalls/futex.S | 25 ---------- libc/arch-mips/bionic/futex_mips.S | 50 -------------------- libc/arch-mips/mips.mk | 1 - libc/arch-mips/syscalls/futex.S | 19 -------- libc/arch-mips64/bionic/futex_mips.S | 64 -------------------------- libc/arch-mips64/mips64.mk | 1 - libc/arch-mips64/syscalls/futex.S | 25 ---------- libc/arch-x86/bionic/futex_x86.S | 16 ------- libc/arch-x86/syscalls/futex.S | 42 ----------------- libc/arch-x86/x86.mk | 1 - libc/arch-x86_64/bionic/futex_x86_64.S | 37 --------------- libc/arch-x86_64/syscalls/futex.S | 17 ------- libc/arch-x86_64/x86_64.mk | 1 - libc/bionic/ndk_cruft.cpp | 20 +++----- libc/bionic/pthread_create.cpp | 20 ++++---- libc/bionic/pthread_internal.h | 2 + libc/private/bionic_futex.h | 23 ++++++--- libc/private/bionic_tls.h | 25 ++++------ 24 files changed, 45 insertions(+), 455 deletions(-) delete mode 100644 libc/arch-arm/bionic/futex_arm.S delete mode 100644 libc/arch-arm/syscalls/futex.S delete mode 100644 libc/arch-arm64/bionic/futex_arm64.S delete mode 100644 libc/arch-arm64/syscalls/futex.S delete mode 100644 libc/arch-mips/bionic/futex_mips.S delete mode 100644 libc/arch-mips/syscalls/futex.S delete mode 100644 libc/arch-mips64/bionic/futex_mips.S delete mode 100644 libc/arch-mips64/syscalls/futex.S delete mode 100644 libc/arch-x86/bionic/futex_x86.S delete mode 100644 libc/arch-x86/syscalls/futex.S delete mode 100644 libc/arch-x86_64/bionic/futex_x86_64.S delete mode 100644 libc/arch-x86_64/syscalls/futex.S diff --git a/libc/SYSCALLS.TXT b/libc/SYSCALLS.TXT index 60cf98019..785d19158 100644 --- a/libc/SYSCALLS.TXT +++ b/libc/SYSCALLS.TXT @@ -305,8 +305,6 @@ int eventfd:eventfd2(unsigned int, int) all void _exit|_Exit:exit_group(int) all void __exit:exit(int) all -int futex(void*, int, int, void*, void*, int) all - int inotify_init1(int) all int inotify_add_watch(int, const char*, unsigned int) all int inotify_rm_watch(int, unsigned int) all diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index 3821854ad..c509b8366 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -53,7 +53,6 @@ libc_bionic_src_files_arm += \ arch-arm/bionic/__bionic_clone.S \ arch-arm/bionic/eabi.c \ arch-arm/bionic/_exit_with_stack_teardown.S \ - arch-arm/bionic/futex_arm.S \ arch-arm/bionic/__get_sp.S \ arch-arm/bionic/libgcc_compat.c \ arch-arm/bionic/memcmp16.S \ diff --git a/libc/arch-arm/bionic/futex_arm.S b/libc/arch-arm/bionic/futex_arm.S deleted file mode 100644 index 89a1e9689..000000000 --- a/libc/arch-arm/bionic/futex_arm.S +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2008 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include - -// int __futex_syscall4(volatile void* ftx, int op, int val, const struct timespec* timeout) -ENTRY_PRIVATE(__futex_syscall4) - mov ip, r7 - ldr r7, =__NR_futex - swi #0 - mov r7, ip - bx lr -END(__futex_syscall4) diff --git a/libc/arch-arm/syscalls/futex.S b/libc/arch-arm/syscalls/futex.S deleted file mode 100644 index 1646ca207..000000000 --- a/libc/arch-arm/syscalls/futex.S +++ /dev/null @@ -1,22 +0,0 @@ -/* Generated by gensyscalls.py. Do not edit. */ - -#include - -ENTRY(futex) - mov ip, sp - stmfd sp!, {r4, r5, r6, r7} - .cfi_def_cfa_offset 16 - .cfi_rel_offset r4, 0 - .cfi_rel_offset r5, 4 - .cfi_rel_offset r6, 8 - .cfi_rel_offset r7, 12 - ldmfd ip, {r4, r5, r6} - ldr r7, =__NR_futex - swi #0 - ldmfd sp!, {r4, r5, r6, r7} - .cfi_def_cfa_offset 0 - cmn r0, #(MAX_ERRNO + 1) - bxls lr - neg r0, r0 - b __set_errno -END(futex) diff --git a/libc/arch-arm64/arm64.mk b/libc/arch-arm64/arm64.mk index 7a9eb4ebd..93b0b0b27 100644 --- a/libc/arch-arm64/arm64.mk +++ b/libc/arch-arm64/arm64.mk @@ -37,7 +37,6 @@ libc_bionic_src_files_arm64 := \ arch-arm64/bionic/__bionic_clone.S \ arch-arm64/bionic/bzero_arm64.c \ arch-arm64/bionic/_exit_with_stack_teardown.S \ - arch-arm64/bionic/futex_arm64.S \ arch-arm64/bionic/__get_sp.S \ arch-arm64/bionic/__rt_sigreturn.S \ arch-arm64/bionic/_setjmp.S \ diff --git a/libc/arch-arm64/bionic/futex_arm64.S b/libc/arch-arm64/bionic/futex_arm64.S deleted file mode 100644 index 9d7465ae4..000000000 --- a/libc/arch-arm64/bionic/futex_arm64.S +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include - -// int __futex_syscall4(volatile void* ftx, int op, int val, const struct timespec* timeout) -ENTRY_PRIVATE(__futex_syscall4) - stp x29, x30, [sp, #-16]! - .cfi_def_cfa_offset 16 - .cfi_rel_offset x29, 0 - .cfi_rel_offset x30, 8 - mov x29, sp - - mov x8, __NR_futex - svc #0 - - ldp x29, x30, [sp], #16 - .cfi_def_cfa_offset 0 - .cfi_restore x29 - .cfi_restore x30 - ret -END(__futex_syscall4) diff --git a/libc/arch-arm64/syscalls/futex.S b/libc/arch-arm64/syscalls/futex.S deleted file mode 100644 index c14ebbff1..000000000 --- a/libc/arch-arm64/syscalls/futex.S +++ /dev/null @@ -1,25 +0,0 @@ -/* Generated by gensyscalls.py. Do not edit. */ - -#include - -ENTRY(futex) - stp x29, x30, [sp, #-16]! - .cfi_def_cfa_offset 16 - .cfi_rel_offset x29, 0 - .cfi_rel_offset x30, 8 - mov x29, sp - - mov x8, __NR_futex - svc #0 - - ldp x29, x30, [sp], #16 - .cfi_def_cfa_offset 0 - .cfi_restore x29 - .cfi_restore x30 - - cmn x0, #(MAX_ERRNO + 1) - cneg x0, x0, hi - b.hi __set_errno - - ret -END(futex) diff --git a/libc/arch-mips/bionic/futex_mips.S b/libc/arch-mips/bionic/futex_mips.S deleted file mode 100644 index 5a09f325d..000000000 --- a/libc/arch-mips/bionic/futex_mips.S +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2008 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include - -// int __futex_syscall4(volatile void* ftx, int op, int val, const struct timespec* timeout) -ENTRY_PRIVATE(__futex_syscall4) - subu sp,4*6 - sw $0,20(sp) /* val3 */ - sw $0,16(sp) /* addr2 */ -# move a3,a3 /* timespec */ -# move a2,a2 /* val */ -# li a1,a1 /* op */ -# move a0,a0 /* ftx */ - li v0,__NR_futex - syscall - .set noreorder - bnez a3, 1f /* Check for error */ - neg v0 /* Negate error number if it's valid */ - move v0,$0 /* Otherwise return 0 */ -1: - .set reorder - addu sp,4*6 - j ra -END(__futex_syscall4) diff --git a/libc/arch-mips/mips.mk b/libc/arch-mips/mips.mk index d7d1df417..2c87a9ee3 100644 --- a/libc/arch-mips/mips.mk +++ b/libc/arch-mips/mips.mk @@ -58,7 +58,6 @@ libc_bionic_src_files_mips += \ arch-mips/bionic/bzero.S \ arch-mips/bionic/cacheflush.cpp \ arch-mips/bionic/_exit_with_stack_teardown.S \ - arch-mips/bionic/futex_mips.S \ arch-mips/bionic/__get_sp.S \ arch-mips/bionic/memcmp16.S \ arch-mips/bionic/_setjmp.S \ diff --git a/libc/arch-mips/syscalls/futex.S b/libc/arch-mips/syscalls/futex.S deleted file mode 100644 index a865fea10..000000000 --- a/libc/arch-mips/syscalls/futex.S +++ /dev/null @@ -1,19 +0,0 @@ -/* Generated by gensyscalls.py. Do not edit. */ - -#include - -ENTRY(futex) - .set noreorder - .cpload t9 - li v0, __NR_futex - syscall - bnez a3, 1f - move a0, v0 - j ra - nop -1: - la t9,__set_errno - j t9 - nop - .set reorder -END(futex) diff --git a/libc/arch-mips64/bionic/futex_mips.S b/libc/arch-mips64/bionic/futex_mips.S deleted file mode 100644 index 60c218c26..000000000 --- a/libc/arch-mips64/bionic/futex_mips.S +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2008 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include - -#if (_MIPS_SIM == _ABIO32) || (_MIPS_SIM == _ABI32) -FRAMESZ = MKFSIZ(NARGSAVE+2,0) -FRAME_A4 = 4*REGSZ -FRAME_A5 = 5*REGSZ -#else -FRAMESZ = 0 -#endif - -// int __futex_syscall4(volatile void* ftx, int op, int val, const struct timespec* timeout) -LEAF(__futex_syscall4,FRAMESZ) -#if (_MIPS_SIM == _ABIO32) || (_MIPS_SIM == _ABI32) - PTR_SUBU sp, FRAMESZ - REG_S $0,FRAME_A5(sp) /* val3 */ - REG_S $0,FRAME_A4(sp) /* addr2 */ -#else - move a5,$0 /* val3 */ - move a4,$0 /* addr2 */ -#endif -# move a3,a3 /* timespec */ -# move a2,a2 /* val */ -# move a1,a1 /* op */ -# move a0,a0 /* ftx */ - LI v0,__NR_futex - syscall - neg v0 /* Negate errno */ - bnez a3,1f /* Check for error */ - move v0,$0 /* Return 0 if no error */ -1: -#if (_MIPS_SIM == _ABIO32) || (_MIPS_SIM == _ABI32) - PTR_ADDU sp,FRAMESZ -#endif - j ra - END(__futex_syscall4) -.hidden __futex_syscall4 diff --git a/libc/arch-mips64/mips64.mk b/libc/arch-mips64/mips64.mk index b6e0209a6..26390a648 100644 --- a/libc/arch-mips64/mips64.mk +++ b/libc/arch-mips64/mips64.mk @@ -43,7 +43,6 @@ libc_bionic_src_files_mips64 := \ arch-mips64/bionic/__bionic_clone.S \ arch-mips64/bionic/bzero.S \ arch-mips64/bionic/_exit_with_stack_teardown.S \ - arch-mips64/bionic/futex_mips.S \ arch-mips64/bionic/__get_sp.S \ arch-mips64/bionic/getdents.cpp \ arch-mips64/bionic/memcmp16.S \ diff --git a/libc/arch-mips64/syscalls/futex.S b/libc/arch-mips64/syscalls/futex.S deleted file mode 100644 index dc7dcc6ee..000000000 --- a/libc/arch-mips64/syscalls/futex.S +++ /dev/null @@ -1,25 +0,0 @@ -/* Generated by gensyscalls.py. Do not edit. */ - -#include - -ENTRY(futex) - .set push - .set noreorder - li v0, __NR_futex - syscall - bnez a3, 1f - move a0, v0 - j ra - nop -1: - move t0, ra - bal 2f - nop -2: - .cpsetup ra, t1, 2b - LA t9,__set_errno - .cpreturn - j t9 - move ra, t0 - .set pop -END(futex) diff --git a/libc/arch-x86/bionic/futex_x86.S b/libc/arch-x86/bionic/futex_x86.S deleted file mode 100644 index 94647cabd..000000000 --- a/libc/arch-x86/bionic/futex_x86.S +++ /dev/null @@ -1,16 +0,0 @@ -#include - -// int __futex_syscall4(volatile void* ftx, int op, int val, const struct timespec* timeout) -ENTRY_PRIVATE(__futex_syscall4) - pushl %ebx - pushl %esi - movl 12(%esp), %ebx /* ftx */ - movl 16(%esp), %ecx /* op */ - movl 20(%esp), %edx /* val */ - movl 24(%esp), %esi /* timeout */ - movl $__NR_futex, %eax - int $0x80 - popl %esi - popl %ebx - ret -END(__futex_syscall4) diff --git a/libc/arch-x86/syscalls/futex.S b/libc/arch-x86/syscalls/futex.S deleted file mode 100644 index 7a5291361..000000000 --- a/libc/arch-x86/syscalls/futex.S +++ /dev/null @@ -1,42 +0,0 @@ -/* Generated by gensyscalls.py. Do not edit. */ - -#include - -ENTRY(futex) - pushl %ebx - pushl %ecx - pushl %edx - pushl %esi - pushl %edi - pushl %ebp - .cfi_def_cfa_offset 24 - .cfi_rel_offset ebx, 0 - .cfi_rel_offset ecx, 4 - .cfi_rel_offset edx, 8 - .cfi_rel_offset esi, 12 - .cfi_rel_offset edi, 16 - .cfi_rel_offset ebp, 20 - mov 28(%esp), %ebx - mov 32(%esp), %ecx - mov 36(%esp), %edx - mov 40(%esp), %esi - mov 44(%esp), %edi - mov 48(%esp), %ebp - movl $__NR_futex, %eax - int $0x80 - cmpl $-MAX_ERRNO, %eax - jb 1f - negl %eax - pushl %eax - call __set_errno - addl $4, %esp - orl $-1, %eax -1: - popl %ebp - popl %edi - popl %esi - popl %edx - popl %ecx - popl %ebx - ret -END(futex) diff --git a/libc/arch-x86/x86.mk b/libc/arch-x86/x86.mk index 34da0ced1..aa183cb83 100644 --- a/libc/arch-x86/x86.mk +++ b/libc/arch-x86/x86.mk @@ -23,7 +23,6 @@ libc_bionic_src_files_x86 := \ libc_bionic_src_files_x86 += \ arch-x86/bionic/__bionic_clone.S \ arch-x86/bionic/_exit_with_stack_teardown.S \ - arch-x86/bionic/futex_x86.S \ arch-x86/bionic/__get_sp.S \ arch-x86/bionic/_setjmp.S \ arch-x86/bionic/setjmp.S \ diff --git a/libc/arch-x86_64/bionic/futex_x86_64.S b/libc/arch-x86_64/bionic/futex_x86_64.S deleted file mode 100644 index c24843930..000000000 --- a/libc/arch-x86_64/bionic/futex_x86_64.S +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include - -// int __futex_syscall4(volatile void* ftx, int op, int val, const struct timespec* timeout) -ENTRY_PRIVATE(__futex_syscall4) - mov %rcx, %r10 /* timeout */ - mov $__NR_futex, %eax - syscall - ret -END(__futex_syscall4) diff --git a/libc/arch-x86_64/syscalls/futex.S b/libc/arch-x86_64/syscalls/futex.S deleted file mode 100644 index 62f64bd9f..000000000 --- a/libc/arch-x86_64/syscalls/futex.S +++ /dev/null @@ -1,17 +0,0 @@ -/* Generated by gensyscalls.py. Do not edit. */ - -#include - -ENTRY(futex) - movq %rcx, %r10 - movl $__NR_futex, %eax - syscall - cmpq $-MAX_ERRNO, %rax - jb 1f - negl %eax - movl %eax, %edi - call __set_errno - orq $-1, %rax -1: - ret -END(futex) diff --git a/libc/arch-x86_64/x86_64.mk b/libc/arch-x86_64/x86_64.mk index 2bcf43258..e3c2562bb 100644 --- a/libc/arch-x86_64/x86_64.mk +++ b/libc/arch-x86_64/x86_64.mk @@ -30,7 +30,6 @@ libc_common_src_files_x86_64 += \ libc_bionic_src_files_x86_64 := \ arch-x86_64/bionic/__bionic_clone.S \ arch-x86_64/bionic/_exit_with_stack_teardown.S \ - arch-x86_64/bionic/futex_x86_64.S \ arch-x86_64/bionic/__get_sp.S \ arch-x86_64/bionic/__rt_sigreturn.S \ arch-x86_64/bionic/_setjmp.S \ diff --git a/libc/bionic/ndk_cruft.cpp b/libc/bionic/ndk_cruft.cpp index 4900a8a9a..1284b9a4e 100644 --- a/libc/bionic/ndk_cruft.cpp +++ b/libc/bionic/ndk_cruft.cpp @@ -31,7 +31,6 @@ #include #include -#include #include #include #include @@ -201,25 +200,20 @@ extern "C" int vfdprintf(int fd, const char* fmt, va_list ap) { return vdprintf(fd, fmt, ap); } -static inline int __futex(volatile void* ftx, int op, int value, const struct timespec* timeout) { - // Our generated syscall assembler sets errno, but our callers (pthread functions) don't want to. - int saved_errno = errno; - if (syscall(__NR_futex, ftx, op, value, timeout) == 0) { - return 0; - } - int result = -errno; - errno = saved_errno; - return result; -} +#define __futex_wake __real_futex_wake +#define __futex_wait __real_futex_wait +#include "private/bionic_futex.h" +#undef __futex_wake +#undef __futex_wait // This used to be in . extern "C" int __futex_wake(volatile void* ftx, int count) { - return __futex(ftx, FUTEX_WAKE, count, NULL); + return __real_futex_wake(ftx, count); } // This used to be in . extern "C" int __futex_wait(volatile void* ftx, int value, const struct timespec* timeout) { - return __futex(ftx, FUTEX_WAIT, value, timeout); + return __real_futex_wait(ftx, value, timeout); } // Unity's libmono uses this. diff --git a/libc/bionic/pthread_create.cpp b/libc/bionic/pthread_create.cpp index 303af8193..c4cb262ac 100644 --- a/libc/bionic/pthread_create.cpp +++ b/libc/bionic/pthread_create.cpp @@ -144,10 +144,8 @@ static int __pthread_start(void* arg) { // notify gdb about this thread before we start doing anything. // This also provides the memory barrier needed to ensure that all memory // accesses previously made by the creating thread are visible to us. - pthread_mutex_t* start_mutex = (pthread_mutex_t*) &thread->tls[TLS_SLOT_START_MUTEX]; - pthread_mutex_lock(start_mutex); - pthread_mutex_destroy(start_mutex); - thread->tls[TLS_SLOT_START_MUTEX] = NULL; + pthread_mutex_lock(&thread->startup_handshake_mutex); + pthread_mutex_destroy(&thread->startup_handshake_mutex); __init_alternate_signal_stack(thread); @@ -204,7 +202,8 @@ int pthread_create(pthread_t* thread_out, pthread_attr_t const* attr, // The child stack is the same address, just growing in the opposite direction. // At offsets >= 0, we have the TLS slots. // At offsets < 0, we have the child stack. - thread->tls = (void**)((uint8_t*)(thread->attr.stack_base) + thread->attr.stack_size - BIONIC_TLS_SLOTS * sizeof(void*)); + thread->tls = reinterpret_cast(reinterpret_cast(thread->attr.stack_base) + + thread->attr.stack_size - BIONIC_TLS_SLOTS * sizeof(void*)); void* child_stack = thread->tls; __init_tls(thread); @@ -214,9 +213,8 @@ int pthread_create(pthread_t* thread_out, pthread_attr_t const* attr, // This also provides the memory barrier we need to ensure that all // memory accesses previously performed by this thread are visible to // the new thread. - pthread_mutex_t* start_mutex = (pthread_mutex_t*) &thread->tls[TLS_SLOT_START_MUTEX]; - pthread_mutex_init(start_mutex, NULL); - pthread_mutex_lock(start_mutex); + pthread_mutex_init(&thread->startup_handshake_mutex, NULL); + pthread_mutex_lock(&thread->startup_handshake_mutex); thread->start_routine = start_routine; thread->start_routine_arg = arg; @@ -237,7 +235,7 @@ int pthread_create(pthread_t* thread_out, pthread_attr_t const* attr, // We don't have to unlock the mutex at all because clone(2) failed so there's no child waiting to // be unblocked, but we're about to unmap the memory the mutex is stored in, so this serves as a // reminder that you can't rewrite this function to use a ScopedPthreadMutexLocker. - pthread_mutex_unlock(start_mutex); + pthread_mutex_unlock(&thread->startup_handshake_mutex); if ((thread->attr.flags & PTHREAD_ATTR_FLAG_USER_ALLOCATED_STACK) == 0) { munmap(thread->attr.stack_base, thread->attr.stack_size); } @@ -252,7 +250,7 @@ int pthread_create(pthread_t* thread_out, pthread_attr_t const* attr, // Letting the thread run is the easiest way to clean up its resources. thread->attr.flags |= PTHREAD_ATTR_FLAG_DETACHED; thread->start_routine = __do_nothing; - pthread_mutex_unlock(start_mutex); + pthread_mutex_unlock(&thread->startup_handshake_mutex); return init_errno; } @@ -264,7 +262,7 @@ int pthread_create(pthread_t* thread_out, pthread_attr_t const* attr, // Publish the pthread_t and unlock the mutex to let the new thread start running. *thread_out = reinterpret_cast(thread); - pthread_mutex_unlock(start_mutex); + pthread_mutex_unlock(&thread->startup_handshake_mutex); return 0; } diff --git a/libc/bionic/pthread_internal.h b/libc/bionic/pthread_internal.h index 295d9d6ae..490ae86d4 100644 --- a/libc/bionic/pthread_internal.h +++ b/libc/bionic/pthread_internal.h @@ -48,6 +48,8 @@ struct pthread_internal_t { void* alternate_signal_stack; + pthread_mutex_t startup_handshake_mutex; + /* * The dynamic linker implements dlerror(3), which makes it hard for us to implement this * per-thread buffer by simply using malloc(3) and free(3). diff --git a/libc/private/bionic_futex.h b/libc/private/bionic_futex.h index 11699cea6..35d33d0bf 100644 --- a/libc/private/bionic_futex.h +++ b/libc/private/bionic_futex.h @@ -28,31 +28,42 @@ #ifndef _BIONIC_FUTEX_H #define _BIONIC_FUTEX_H +#include #include -#include #include #include +#include +#include __BEGIN_DECLS struct timespec; -extern int __futex_syscall4(volatile void* ftx, int op, int value, const struct timespec* timeout); +static inline __always_inline int __futex(volatile void* ftx, int op, int value, const struct timespec* timeout) { + // Our generated syscall assembler sets errno, but our callers (pthread functions) don't want to. + int saved_errno = errno; + if (__predict_false(syscall(__NR_futex, ftx, op, value, timeout) == -1)) { + int result = -errno; + errno = saved_errno; + return result; + } + return 0; +} static inline int __futex_wake(volatile void* ftx, int count) { - return __futex_syscall4(ftx, FUTEX_WAKE, count, NULL); + return __futex(ftx, FUTEX_WAKE, count, NULL); } static inline int __futex_wake_ex(volatile void* ftx, bool shared, int count) { - return __futex_syscall4(ftx, shared ? FUTEX_WAKE : FUTEX_WAKE_PRIVATE, count, NULL); + return __futex(ftx, shared ? FUTEX_WAKE : FUTEX_WAKE_PRIVATE, count, NULL); } static inline int __futex_wait(volatile void* ftx, int value, const struct timespec* timeout) { - return __futex_syscall4(ftx, FUTEX_WAIT, value, timeout); + return __futex(ftx, FUTEX_WAIT, value, timeout); } static inline int __futex_wait_ex(volatile void* ftx, bool shared, int value, const struct timespec* timeout) { - return __futex_syscall4(ftx, shared ? FUTEX_WAIT : FUTEX_WAIT_PRIVATE, value, timeout); + return __futex(ftx, shared ? FUTEX_WAIT : FUTEX_WAIT_PRIVATE, value, timeout); } __END_DECLS diff --git a/libc/private/bionic_tls.h b/libc/private/bionic_tls.h index c2cf196d6..b52013f5c 100644 --- a/libc/private/bionic_tls.h +++ b/libc/private/bionic_tls.h @@ -46,32 +46,27 @@ __BEGIN_DECLS ** pre-allocated slot directly for performance reason). **/ -/* Well-known TLS slots. What data goes in which slot is arbitrary unless otherwise noted. */ +// Well-known TLS slots. What data goes in which slot is arbitrary unless otherwise noted. enum { - TLS_SLOT_SELF = 0, /* The kernel requires this specific slot for x86. */ + TLS_SLOT_SELF = 0, // The kernel requires this specific slot for x86. TLS_SLOT_THREAD_ID, TLS_SLOT_ERRNO, - /* This slot in the child's TLS is used to synchronize the parent and child - * during thread initialization. The child finishes with this mutex before - * running any code that can set errno, so we can reuse the errno slot. */ - TLS_SLOT_START_MUTEX = TLS_SLOT_ERRNO, - - /* These two aren't used by bionic itself, but allow the graphics code to - * access TLS directly rather than using the pthread API. */ + // These two aren't used by bionic itself, but allow the graphics code to + // access TLS directly rather than using the pthread API. TLS_SLOT_OPENGL_API = 3, TLS_SLOT_OPENGL = 4, - /* This slot is only used to pass information from the dynamic linker to - * libc.so when the C library is loaded in to memory. The C runtime init - * function will then clear it. Since its use is extremely temporary, - * we reuse an existing location that isn't needed during libc startup. */ + // This slot is only used to pass information from the dynamic linker to + // libc.so when the C library is loaded in to memory. The C runtime init + // function will then clear it. Since its use is extremely temporary, + // we reuse an existing location that isn't needed during libc startup. TLS_SLOT_BIONIC_PREINIT = TLS_SLOT_OPENGL_API, - TLS_SLOT_STACK_GUARD = 5, /* GCC requires this specific slot for x86. */ + TLS_SLOT_STACK_GUARD = 5, // GCC requires this specific slot for x86. TLS_SLOT_DLERROR, - TLS_SLOT_FIRST_USER_SLOT /* Must come last! */ + TLS_SLOT_FIRST_USER_SLOT // Must come last! }; /*