From acdde8c1cf8e8beed98c052757d96695b820b50c Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Tue, 26 Feb 2013 01:30:00 -0800 Subject: [PATCH 01/77] Break bionic implementations into arch versions. Move arch specific code for arm, mips, x86 into separate makefiles. In addition, add different arm cpu versions of memcpy/memset. Bug: 8005082 Change-Id: I04f3d0715104fab618e1abf7cf8f7eec9bec79df --- libc/Android.mk | 116 ++----- libc/arch-arm/arm.mk | 36 +++ libc/arch-arm/cortex-a15/bionic/memcpy.S | 146 +++++++++ libc/arch-arm/cortex-a15/bionic/memset.S | 106 +++++++ libc/arch-arm/cortex-a15/cortex-a15.mk | 4 + libc/arch-arm/cortex-a9/bionic/memcpy.S | 211 +++++++++++++ libc/arch-arm/cortex-a9/bionic/memset.S | 152 +++++++++ libc/arch-arm/cortex-a9/cortex-a9.mk | 4 + libc/arch-arm/generic/bionic/memcpy.S | 380 +++++++++++++++++++++++ libc/arch-arm/generic/bionic/memset.S | 109 +++++++ libc/arch-arm/generic/generic.mk | 2 + libc/arch-arm/krait/bionic/memcpy.S | 146 +++++++++ libc/arch-arm/krait/bionic/memset.S | 81 +++++ libc/arch-arm/krait/krait.mk | 4 + libc/arch-mips/mips.mk | 25 ++ libc/arch-x86/x86.mk | 27 ++ 16 files changed, 1467 insertions(+), 82 deletions(-) create mode 100644 libc/arch-arm/arm.mk create mode 100644 libc/arch-arm/cortex-a15/bionic/memcpy.S create mode 100644 libc/arch-arm/cortex-a15/bionic/memset.S create mode 100644 libc/arch-arm/cortex-a15/cortex-a15.mk create mode 100644 libc/arch-arm/cortex-a9/bionic/memcpy.S create mode 100644 libc/arch-arm/cortex-a9/bionic/memset.S create mode 100644 libc/arch-arm/cortex-a9/cortex-a9.mk create mode 100644 libc/arch-arm/generic/bionic/memcpy.S create mode 100644 libc/arch-arm/generic/bionic/memset.S create mode 100644 libc/arch-arm/generic/generic.mk create mode 100644 libc/arch-arm/krait/bionic/memcpy.S create mode 100644 libc/arch-arm/krait/bionic/memset.S create mode 100644 libc/arch-arm/krait/krait.mk create mode 100644 libc/arch-mips/mips.mk create mode 100644 libc/arch-x86/x86.mk diff --git a/libc/Android.mk b/libc/Android.mk index cd62dcd5f..9a55eff27 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -364,29 +364,6 @@ libc_upstream_netbsd_src_files := \ # ========================================================= ifeq ($(TARGET_ARCH),arm) libc_common_src_files += \ - arch-arm/bionic/abort_arm.S \ - arch-arm/bionic/atomics_arm.c \ - arch-arm/bionic/clone.S \ - arch-arm/bionic/eabi.c \ - arch-arm/bionic/_exit_with_stack_teardown.S \ - arch-arm/bionic/ffs.S \ - arch-arm/bionic/futex_arm.S \ - arch-arm/bionic/__get_sp.S \ - arch-arm/bionic/kill.S \ - arch-arm/bionic/libgcc_compat.c \ - arch-arm/bionic/memcmp16.S \ - arch-arm/bionic/memcmp.S \ - arch-arm/bionic/memcpy.S \ - arch-arm/bionic/memset.S \ - arch-arm/bionic/_setjmp.S \ - arch-arm/bionic/setjmp.S \ - arch-arm/bionic/sigsetjmp.S \ - arch-arm/bionic/strcmp.S \ - arch-arm/bionic/strcpy.S \ - arch-arm/bionic/strlen.c.arm \ - arch-arm/bionic/syscall.S \ - arch-arm/bionic/tgkill.S \ - arch-arm/bionic/tkill.S \ bionic/memmove.c.arm \ bionic/socketcalls.c \ string/bcopy.c \ @@ -406,38 +383,10 @@ libc_static_common_src_files += \ bionic/pthread_create.cpp.arm \ bionic/pthread_key.cpp.arm \ -# these are used by the static and dynamic versions of the libc -# respectively -libc_arch_static_src_files := \ - arch-arm/bionic/exidx_static.c - -libc_arch_dynamic_src_files := \ - arch-arm/bionic/exidx_dynamic.c endif # arm ifeq ($(TARGET_ARCH),x86) libc_common_src_files += \ - arch-x86/bionic/clone.S \ - arch-x86/bionic/_exit_with_stack_teardown.S \ - arch-x86/bionic/futex_x86.S \ - arch-x86/bionic/__get_sp.S \ - arch-x86/bionic/__get_tls.c \ - arch-x86/bionic/_setjmp.S \ - arch-x86/bionic/setjmp.S \ - arch-x86/bionic/__set_tls.c \ - arch-x86/bionic/sigsetjmp.S \ - arch-x86/bionic/syscall.S \ - arch-x86/bionic/vfork.S \ - arch-x86/string/bcopy_wrapper.S \ - arch-x86/string/bzero_wrapper.S \ - arch-x86/string/ffs.S \ - arch-x86/string/memcmp_wrapper.S \ - arch-x86/string/memcpy_wrapper.S \ - arch-x86/string/memmove_wrapper.S \ - arch-x86/string/memset_wrapper.S \ - arch-x86/string/strcmp_wrapper.S \ - arch-x86/string/strlen_wrapper.S \ - arch-x86/string/strncmp_wrapper.S \ bionic/pthread-atfork.c \ bionic/pthread-rwlocks.c \ bionic/pthread-timers.c \ @@ -449,36 +398,9 @@ libc_static_common_src_files += \ bionic/pthread_create.cpp \ bionic/pthread_key.cpp \ -libc_arch_static_src_files := \ - bionic/dl_iterate_phdr_static.c - -libc_arch_dynamic_src_files := endif # x86 ifeq ($(TARGET_ARCH),mips) -libc_common_src_files += \ - arch-mips/bionic/__get_sp.S \ - arch-mips/bionic/__get_tls.c \ - arch-mips/bionic/__set_tls.c \ - arch-mips/bionic/_exit_with_stack_teardown.S \ - arch-mips/bionic/_setjmp.S \ - arch-mips/bionic/futex_mips.S \ - arch-mips/bionic/bzero.S \ - arch-mips/bionic/cacheflush.c \ - arch-mips/bionic/clone.S \ - arch-mips/bionic/ffs.S \ - arch-mips/bionic/memcmp16.S \ - arch-mips/bionic/memmove.c \ - arch-mips/bionic/pipe.S \ - arch-mips/bionic/setjmp.S \ - arch-mips/bionic/sigsetjmp.S \ - arch-mips/bionic/vfork.S - -libc_common_src_files += \ - arch-mips/string/memset.S \ - arch-mips/string/memcpy.S \ - arch-mips/string/mips_strlen.c - libc_common_src_files += \ bionic/memcmp.c \ string/bcopy.c \ @@ -497,12 +419,42 @@ libc_static_common_src_files += \ bionic/pthread_create.cpp \ bionic/pthread_key.cpp \ -libc_arch_static_src_files := \ - bionic/dl_iterate_phdr_static.c - -libc_arch_dynamic_src_files := endif # mips +ifeq ($(strip $(TARGET_CPU_VARIANT)),) +$(warning TARGET_CPU_VARIANT is not defined) +endif + +########################################################### +## Add cpu specific source files. +## +## This can be called multiple times, but it will only add +## the first source file for each unique $(1). +## $(1): Unique identifier to identify the cpu variant +## implementation. +## $(2): Cpu specific source file. +########################################################### + +define libc-add-cpu-variant-src +$(if $(filter true,$(_LIBC_ARCH_CPU_VARIANT_HAS_$(1))), \ + , \ + $(eval _LIBC_ARCH_CPU_VARIANT_HAS_$(1) := true) \ + $(eval _LIBC_ARCH_CPU_VARIANT_SRC_FILE.$(1) := $(2)) \ + $(eval _LIBC_ARCH_CPU_VARIANT_SRC_FILES += $(2)) \ +) +endef + +_LIBC_ARCH_COMMON_SRC_FILES := +_LIBC_ARCH_CPU_VARIANT_SRC_FILES := +_LIBC_ARCH_STATIC_SRC_FILES := +_LIBC_ARCH_DYNAMIC_SRC_FILES := +include bionic/libc/arch-$(TARGET_ARCH)/$(TARGET_ARCH).mk + +libc_common_src_files += $(_LIBC_ARCH_COMMON_SRC_FILES) +libc_common_src_files += $(_LIBC_ARCH_CPU_VARIANT_SRC_FILES) +libc_arch_static_src_files := $(_LIBC_ARCH_STATIC_SRC_FILES) +libc_arch_dynamic_src_files := $(_LIBC_ARCH_DYNAMIC_SRC_FILES) + # Define some common cflags # ======================================================== libc_common_cflags := \ diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk new file mode 100644 index 000000000..2def30b17 --- /dev/null +++ b/libc/arch-arm/arm.mk @@ -0,0 +1,36 @@ +_LIBC_ARCH_COMMON_SRC_FILES := \ + arch-arm/bionic/abort_arm.S \ + arch-arm/bionic/atomics_arm.c \ + arch-arm/bionic/clone.S \ + arch-arm/bionic/eabi.c \ + arch-arm/bionic/_exit_with_stack_teardown.S \ + arch-arm/bionic/ffs.S \ + arch-arm/bionic/futex_arm.S \ + arch-arm/bionic/__get_sp.S \ + arch-arm/bionic/kill.S \ + arch-arm/bionic/libgcc_compat.c \ + arch-arm/bionic/memcmp16.S \ + arch-arm/bionic/memcmp.S \ + arch-arm/bionic/_setjmp.S \ + arch-arm/bionic/setjmp.S \ + arch-arm/bionic/sigsetjmp.S \ + arch-arm/bionic/strcmp.S \ + arch-arm/bionic/strcpy.S \ + arch-arm/bionic/strlen.c.arm \ + arch-arm/bionic/syscall.S \ + arch-arm/bionic/tgkill.S \ + arch-arm/bionic/tkill.S \ + +# These are used by the static and dynamic versions of the libc +# respectively. +_LIBC_ARCH_STATIC_SRC_FILES := \ + arch-arm/bionic/exidx_static.c + +_LIBC_ARCH_DYNAMIC_SRC_FILES := \ + arch-arm/bionic/exidx_dynamic.c + +ifeq ($(strip $(wildcard bionic/libc/arch-arm/$(TARGET_CPU_VARIANT)/$(TARGET_CPU_VARIANT).mk)),) +$(error "TARGET_CPU_VARIANT not set or set to an unknown value. Possible values are cortex-a9, cortex-a15, krait. Use generic for devices that do not have a CPU similar to any of the supported cpu variants.") +endif + +include bionic/libc/arch-arm/$(TARGET_CPU_VARIANT)/$(TARGET_CPU_VARIANT).mk diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy.S b/libc/arch-arm/cortex-a15/bionic/memcpy.S new file mode 100644 index 000000000..16187b562 --- /dev/null +++ b/libc/arch-arm/cortex-a15/bionic/memcpy.S @@ -0,0 +1,146 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* Assumes neon instructions and a cache line size of 64 bytes. */ + +#include +#include + +/* + * This code assumes it is running on a processor that supports all arm v7 + * instructions, that supports neon instructions, and that has a 64 byte + * cache line. + */ + + .text + .fpu neon + +#define CACHE_LINE_SIZE 64 + +ENTRY(memcpy) + .save {r0, lr} + /* start preloading as early as possible */ + pld [r1, #(CACHE_LINE_SIZE*0)] + stmfd sp!, {r0, lr} + pld [r1, #(CACHE_LINE_SIZE*1)] + + /* do we have at least 16-bytes to copy (needed for alignment below) */ + cmp r2, #16 + blo 5f + + /* align destination to cache-line for the write-buffer */ + rsb r3, r0, #0 + ands r3, r3, #0xF + beq 0f + + /* copy up to 15-bytes (count in r3) */ + sub r2, r2, r3 + movs ip, r3, lsl #31 + ldrmib lr, [r1], #1 + strmib lr, [r0], #1 + ldrcsb ip, [r1], #1 + ldrcsb lr, [r1], #1 + strcsb ip, [r0], #1 + strcsb lr, [r0], #1 + movs ip, r3, lsl #29 + bge 1f + // copies 4 bytes, destination 32-bits aligned + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]! +1: bcc 2f + // copies 8 bytes, destination 64-bits aligned + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0, :64]! +2: + +0: /* preload immediately the next cache line, which we may need */ + pld [r1, #(CACHE_LINE_SIZE*0)] + pld [r1, #(CACHE_LINE_SIZE*1)] + + /* make sure we have at least 64 bytes to copy */ + subs r2, r2, #64 + blo 2f + + /* Preload all the cache lines we need. + * NOTE: The number of pld below depends on CACHE_LINE_SIZE, + * ideally we would increase the distance in the main loop to + * avoid the goofy code below. In practice this doesn't seem to make + * a big difference. + * NOTE: The value CACHE_LINE_SIZE * 4 was chosen through + * experimentation. + */ + pld [r1, #(CACHE_LINE_SIZE*2)] + pld [r1, #(CACHE_LINE_SIZE*3)] + pld [r1, #(CACHE_LINE_SIZE*4)] + +1: /* The main loop copies 64 bytes at a time */ + vld1.8 {d0 - d3}, [r1]! + vld1.8 {d4 - d7}, [r1]! + pld [r1, #(CACHE_LINE_SIZE*4)] + subs r2, r2, #64 + vst1.8 {d0 - d3}, [r0, :128]! + vst1.8 {d4 - d7}, [r0, :128]! + bhs 1b + +2: /* fix-up the remaining count and make sure we have >= 32 bytes left */ + add r2, r2, #64 + subs r2, r2, #32 + blo 4f + +3: /* 32 bytes at a time. These cache lines were already preloaded */ + vld1.8 {d0 - d3}, [r1]! + subs r2, r2, #32 + vst1.8 {d0 - d3}, [r0, :128]! + bhs 3b +4: /* less than 32 left */ + add r2, r2, #32 + tst r2, #0x10 + beq 5f + // copies 16 bytes, 128-bits aligned + vld1.8 {d0, d1}, [r1]! + vst1.8 {d0, d1}, [r0, :128]! + +5: /* copy up to 15-bytes (count in r2) */ + movs ip, r2, lsl #29 + bcc 1f + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0]! +1: bge 2f + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]! +2: movs ip, r2, lsl #31 + ldrmib r3, [r1], #1 + ldrcsb ip, [r1], #1 + ldrcsb lr, [r1], #1 + strmib r3, [r0], #1 + strcsb ip, [r0], #1 + strcsb lr, [r0], #1 + + ldmfd sp!, {r0, lr} + bx lr +END(memcpy) diff --git a/libc/arch-arm/cortex-a15/bionic/memset.S b/libc/arch-arm/cortex-a15/bionic/memset.S new file mode 100644 index 000000000..7bb329752 --- /dev/null +++ b/libc/arch-arm/cortex-a15/bionic/memset.S @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include + + /* + * Optimized memset() for ARM. + * + * memset() returns its first argument. + */ + + .fpu neon + +ENTRY(bzero) + mov r2, r1 + mov r1, #0 + // Fall through to memset... +END(bzero) + +ENTRY(memset) + .save {r0} + stmfd sp!, {r0} + + vdup.8 q0, r1 + + /* do we have at least 16-bytes to write (needed for alignment below) */ + cmp r2, #16 + blo 3f + + /* align destination to 16 bytes for the write-buffer */ + rsb r3, r0, #0 + ands r3, r3, #0xF + beq 2f + + /* write up to 15-bytes (count in r3) */ + sub r2, r2, r3 + movs ip, r3, lsl #31 + strmib r1, [r0], #1 + strcsb r1, [r0], #1 + strcsb r1, [r0], #1 + movs ip, r3, lsl #29 + bge 1f + + // writes 4 bytes, 32-bits aligned + vst1.32 {d0[0]}, [r0, :32]! +1: bcc 2f + + // writes 8 bytes, 64-bits aligned + vst1.8 {d0}, [r0, :64]! +2: + /* make sure we have at least 32 bytes to write */ + subs r2, r2, #32 + blo 2f + vmov q1, q0 + +1: /* The main loop writes 32 bytes at a time */ + subs r2, r2, #32 + vst1.8 {d0 - d3}, [r0, :128]! + bhs 1b + +2: /* less than 32 left */ + add r2, r2, #32 + tst r2, #0x10 + beq 3f + + // writes 16 bytes, 128-bits aligned + vst1.8 {d0, d1}, [r0, :128]! +3: /* write up to 15-bytes (count in r2) */ + movs ip, r2, lsl #29 + bcc 1f + vst1.8 {d0}, [r0]! +1: bge 2f + vst1.32 {d0[0]}, [r0]! +2: movs ip, r2, lsl #31 + strmib r1, [r0], #1 + strcsb r1, [r0], #1 + strcsb r1, [r0], #1 + ldmfd sp!, {r0} + bx lr +END(memset) diff --git a/libc/arch-arm/cortex-a15/cortex-a15.mk b/libc/arch-arm/cortex-a15/cortex-a15.mk new file mode 100644 index 000000000..4534808c6 --- /dev/null +++ b/libc/arch-arm/cortex-a15/cortex-a15.mk @@ -0,0 +1,4 @@ +$(call libc-add-cpu-variant-src,MEMCPY,arch-arm/cortex-a15/bionic/memcpy.S) +$(call libc-add-cpu-variant-src,MEMSET,arch-arm/cortex-a15/bionic/memset.S) + +include bionic/libc/arch-arm/generic/generic.mk diff --git a/libc/arch-arm/cortex-a9/bionic/memcpy.S b/libc/arch-arm/cortex-a9/bionic/memcpy.S new file mode 100644 index 000000000..70e27b041 --- /dev/null +++ b/libc/arch-arm/cortex-a9/bionic/memcpy.S @@ -0,0 +1,211 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include + +/* + * This code assumes it is running on a processor that supports all arm v7 + * instructions, that supports neon instructions, and that has a 32 byte + * cache line. + */ + + .text + .fpu neon + +#define CACHE_LINE_SIZE 32 + +ENTRY(memcpy) + .save {r0, lr} + /* start preloading as early as possible */ + pld [r1, #(CACHE_LINE_SIZE * 0)] + stmfd sp!, {r0, lr} + pld [r1, #(CACHE_LINE_SIZE * 2)] + + // Check so divider is at least 16 bytes, needed for alignment code. + cmp r2, #16 + blo 5f + + + /* check if buffers are aligned. If so, run arm-only version */ + eor r3, r0, r1 + ands r3, r3, #0x3 + beq 11f + + /* Check the upper size limit for Neon unaligned memory access in memcpy */ + cmp r2, #224 + blo 3f + + /* align destination to 16 bytes for the write-buffer */ + rsb r3, r0, #0 + ands r3, r3, #0xF + beq 3f + + /* copy up to 15-bytes (count in r3) */ + sub r2, r2, r3 + movs ip, r3, lsl #31 + ldrmib lr, [r1], #1 + strmib lr, [r0], #1 + ldrcsb ip, [r1], #1 + ldrcsb lr, [r1], #1 + strcsb ip, [r0], #1 + strcsb lr, [r0], #1 + movs ip, r3, lsl #29 + bge 1f + // copies 4 bytes, destination 32-bits aligned + vld1.32 {d0[0]}, [r1]! + vst1.32 {d0[0]}, [r0, :32]! +1: bcc 2f + // copies 8 bytes, destination 64-bits aligned + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0, :64]! +2: + /* preload immediately the next cache line, which we may need */ + pld [r1, #(CACHE_LINE_SIZE * 0)] + pld [r1, #(CACHE_LINE_SIZE * 2)] +3: + /* make sure we have at least 64 bytes to copy */ + subs r2, r2, #64 + blo 2f + + /* preload all the cache lines we need */ + pld [r1, #(CACHE_LINE_SIZE * 4)] + pld [r1, #(CACHE_LINE_SIZE * 6)] + +1: /* The main loop copies 64 bytes at a time */ + vld1.8 {d0 - d3}, [r1]! + vld1.8 {d4 - d7}, [r1]! + pld [r1, #(CACHE_LINE_SIZE * 6)] + subs r2, r2, #64 + vst1.8 {d0 - d3}, [r0]! + vst1.8 {d4 - d7}, [r0]! + bhs 1b + +2: /* fix-up the remaining count and make sure we have >= 32 bytes left */ + add r2, r2, #64 + subs r2, r2, #32 + blo 4f + +3: /* 32 bytes at a time. These cache lines were already preloaded */ + vld1.8 {d0 - d3}, [r1]! + subs r2, r2, #32 + vst1.8 {d0 - d3}, [r0]! + bhs 3b + +4: /* less than 32 left */ + add r2, r2, #32 + tst r2, #0x10 + beq 5f + // copies 16 bytes, 128-bits aligned + vld1.8 {d0, d1}, [r1]! + vst1.8 {d0, d1}, [r0]! +5: /* copy up to 15-bytes (count in r2) */ + movs ip, r2, lsl #29 + bcc 1f + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0]! +1: bge 2f + vld1.32 {d0[0]}, [r1]! + vst1.32 {d0[0]}, [r0]! +2: movs ip, r2, lsl #31 + ldrmib r3, [r1], #1 + ldrcsb ip, [r1], #1 + ldrcsb lr, [r1], #1 + strmib r3, [r0], #1 + strcsb ip, [r0], #1 + strcsb lr, [r0], #1 + + ldmfd sp!, {r0, lr} + bx lr +11: + /* Simple arm-only copy loop to handle aligned copy operations */ + stmfd sp!, {r4, r5, r6, r7, r8} + pld [r1, #(CACHE_LINE_SIZE * 4)] + + /* Check alignment */ + rsb r3, r1, #0 + ands r3, #3 + beq 2f + + /* align source to 32 bits. We need to insert 2 instructions between + * a ldr[b|h] and str[b|h] because byte and half-word instructions + * stall 2 cycles. + */ + movs r12, r3, lsl #31 + sub r2, r2, r3 /* we know that r3 <= r2 because r2 >= 4 */ + ldrmib r3, [r1], #1 + ldrcsb r4, [r1], #1 + ldrcsb r5, [r1], #1 + strmib r3, [r0], #1 + strcsb r4, [r0], #1 + strcsb r5, [r0], #1 + +2: + subs r2, r2, #64 + blt 4f + +3: /* Main copy loop, copying 64 bytes at a time */ + pld [r1, #(CACHE_LINE_SIZE * 8)] + ldmia r1!, {r3, r4, r5, r6, r7, r8, r12, lr} + stmia r0!, {r3, r4, r5, r6, r7, r8, r12, lr} + ldmia r1!, {r3, r4, r5, r6, r7, r8, r12, lr} + stmia r0!, {r3, r4, r5, r6, r7, r8, r12, lr} + subs r2, r2, #64 + bge 3b + +4: /* Check if there are > 32 bytes left */ + adds r2, r2, #64 + subs r2, r2, #32 + blt 5f + + /* Copy 32 bytes */ + ldmia r1!, {r3, r4, r5, r6, r7, r8, r12, lr} + stmia r0!, {r3, r4, r5, r6, r7, r8, r12, lr} + subs r2, #32 + +5: /* Handle any remaining bytes */ + adds r2, #32 + beq 6f + + movs r12, r2, lsl #28 + ldmcsia r1!, {r3, r4, r5, r6} /* 16 bytes */ + ldmmiia r1!, {r7, r8} /* 8 bytes */ + stmcsia r0!, {r3, r4, r5, r6} + stmmiia r0!, {r7, r8} + movs r12, r2, lsl #30 + ldrcs r3, [r1], #4 /* 4 bytes */ + ldrmih r4, [r1], #2 /* 2 bytes */ + strcs r3, [r0], #4 + strmih r4, [r0], #2 + tst r2, #0x1 + ldrneb r3, [r1] /* last byte */ + strneb r3, [r0] +6: + ldmfd sp!, {r4, r5, r6, r7, r8} + ldmfd sp!, {r0, pc} +END(memcpy) diff --git a/libc/arch-arm/cortex-a9/bionic/memset.S b/libc/arch-arm/cortex-a9/bionic/memset.S new file mode 100644 index 000000000..b58aa456c --- /dev/null +++ b/libc/arch-arm/cortex-a9/bionic/memset.S @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include + +/* + * This code assumes it is running on a processor that supports all arm v7 + * instructions and that supports neon instructions. + */ + + .fpu neon + +ENTRY(bzero) + mov r2, r1 + mov r1, #0 +END(bzero) + +/* memset() returns its first argument. */ +ENTRY(memset) + # The neon memset only wins for less than 132. + cmp r2, #132 + bhi 11f + + .save {r0} + stmfd sp!, {r0} + + vdup.8 q0, r1 + + /* make sure we have at least 32 bytes to write */ + subs r2, r2, #32 + blo 2f + vmov q1, q0 + +1: /* The main loop writes 32 bytes at a time */ + subs r2, r2, #32 + vst1.8 {d0 - d3}, [r0]! + bhs 1b + +2: /* less than 32 left */ + add r2, r2, #32 + tst r2, #0x10 + beq 3f + + // writes 16 bytes, 128-bits aligned + vst1.8 {d0, d1}, [r0]! +3: /* write up to 15-bytes (count in r2) */ + movs ip, r2, lsl #29 + bcc 1f + vst1.8 {d0}, [r0]! +1: bge 2f + vst1.32 {d0[0]}, [r0]! +2: movs ip, r2, lsl #31 + strmib r1, [r0], #1 + strcsb r1, [r0], #1 + strcsb r1, [r0], #1 + ldmfd sp!, {r0} + bx lr +11: + /* compute the offset to align the destination + * offset = (4-(src&3))&3 = -src & 3 + */ + + .save {r0, r4-r7, lr} + stmfd sp!, {r0, r4-r7, lr} + rsb r3, r0, #0 + ands r3, r3, #3 + cmp r3, r2 + movhi r3, r2 + + /* splat r1 */ + mov r1, r1, lsl #24 + orr r1, r1, r1, lsr #8 + orr r1, r1, r1, lsr #16 + + movs r12, r3, lsl #31 + strcsb r1, [r0], #1 /* can't use strh (alignment unknown) */ + strcsb r1, [r0], #1 + strmib r1, [r0], #1 + subs r2, r2, r3 + ldmlsfd sp!, {r0, r4-r7, lr} /* return */ + bxls lr + + /* align the destination to a cache-line */ + mov r12, r1 + mov lr, r1 + mov r4, r1 + mov r5, r1 + mov r6, r1 + mov r7, r1 + + rsb r3, r0, #0 + ands r3, r3, #0x1C + beq 3f + cmp r3, r2 + andhi r3, r2, #0x1C + sub r2, r2, r3 + + /* conditionally writes 0 to 7 words (length in r3) */ + movs r3, r3, lsl #28 + stmcsia r0!, {r1, lr} + stmcsia r0!, {r1, lr} + stmmiia r0!, {r1, lr} + movs r3, r3, lsl #2 + strcs r1, [r0], #4 + +3: + subs r2, r2, #32 + mov r3, r1 + bmi 2f +1: subs r2, r2, #32 + stmia r0!, {r1,r3,r4,r5,r6,r7,r12,lr} + bhs 1b +2: add r2, r2, #32 + + /* conditionally stores 0 to 31 bytes */ + movs r2, r2, lsl #28 + stmcsia r0!, {r1,r3,r12,lr} + stmmiia r0!, {r1, lr} + movs r2, r2, lsl #2 + strcs r1, [r0], #4 + strmih r1, [r0], #2 + movs r2, r2, lsl #2 + strcsb r1, [r0] + ldmfd sp!, {r0, r4-r7, lr} + bx lr +END(memset) diff --git a/libc/arch-arm/cortex-a9/cortex-a9.mk b/libc/arch-arm/cortex-a9/cortex-a9.mk new file mode 100644 index 000000000..91664133b --- /dev/null +++ b/libc/arch-arm/cortex-a9/cortex-a9.mk @@ -0,0 +1,4 @@ +$(call libc-add-cpu-variant-src,MEMCPY,arch-arm/cortex-a9/bionic/memcpy.S) +$(call libc-add-cpu-variant-src,MEMSET,arch-arm/cortex-a9/bionic/memset.S) + +include bionic/libc/arch-arm/generic/generic.mk diff --git a/libc/arch-arm/generic/bionic/memcpy.S b/libc/arch-arm/generic/bionic/memcpy.S new file mode 100644 index 000000000..6890a5502 --- /dev/null +++ b/libc/arch-arm/generic/bionic/memcpy.S @@ -0,0 +1,380 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include + + /* + * Optimized memcpy() for ARM. + * + * note that memcpy() always returns the destination pointer, + * so we have to preserve R0. + */ + +ENTRY(memcpy) + /* The stack must always be 64-bits aligned to be compliant with the + * ARM ABI. Since we have to save R0, we might as well save R4 + * which we can use for better pipelining of the reads below + */ + .save {r0, r4, lr} + stmfd sp!, {r0, r4, lr} + /* Making room for r5-r11 which will be spilled later */ + .pad #28 + sub sp, sp, #28 + + // preload the destination because we'll align it to a cache line + // with small writes. Also start the source "pump". + PLD (r0, #0) + PLD (r1, #0) + PLD (r1, #32) + + /* it simplifies things to take care of len<4 early */ + cmp r2, #4 + blo copy_last_3_and_return + + /* compute the offset to align the source + * offset = (4-(src&3))&3 = -src & 3 + */ + rsb r3, r1, #0 + ands r3, r3, #3 + beq src_aligned + + /* align source to 32 bits. We need to insert 2 instructions between + * a ldr[b|h] and str[b|h] because byte and half-word instructions + * stall 2 cycles. + */ + movs r12, r3, lsl #31 + sub r2, r2, r3 /* we know that r3 <= r2 because r2 >= 4 */ + ldrmib r3, [r1], #1 + ldrcsb r4, [r1], #1 + ldrcsb r12,[r1], #1 + strmib r3, [r0], #1 + strcsb r4, [r0], #1 + strcsb r12,[r0], #1 + +src_aligned: + + /* see if src and dst are aligned together (congruent) */ + eor r12, r0, r1 + tst r12, #3 + bne non_congruent + + /* Use post-incriment mode for stm to spill r5-r11 to reserved stack + * frame. Don't update sp. + */ + stmea sp, {r5-r11} + + /* align the destination to a cache-line */ + rsb r3, r0, #0 + ands r3, r3, #0x1C + beq congruent_aligned32 + cmp r3, r2 + andhi r3, r2, #0x1C + + /* conditionally copies 0 to 7 words (length in r3) */ + movs r12, r3, lsl #28 + ldmcsia r1!, {r4, r5, r6, r7} /* 16 bytes */ + ldmmiia r1!, {r8, r9} /* 8 bytes */ + stmcsia r0!, {r4, r5, r6, r7} + stmmiia r0!, {r8, r9} + tst r3, #0x4 + ldrne r10,[r1], #4 /* 4 bytes */ + strne r10,[r0], #4 + sub r2, r2, r3 + +congruent_aligned32: + /* + * here source is aligned to 32 bytes. + */ + +cached_aligned32: + subs r2, r2, #32 + blo less_than_32_left + + /* + * We preload a cache-line up to 64 bytes ahead. On the 926, this will + * stall only until the requested world is fetched, but the linefill + * continues in the the background. + * While the linefill is going, we write our previous cache-line + * into the write-buffer (which should have some free space). + * When the linefill is done, the writebuffer will + * start dumping its content into memory + * + * While all this is going, we then load a full cache line into + * 8 registers, this cache line should be in the cache by now + * (or partly in the cache). + * + * This code should work well regardless of the source/dest alignment. + * + */ + + // Align the preload register to a cache-line because the cpu does + // "critical word first" (the first word requested is loaded first). + bic r12, r1, #0x1F + add r12, r12, #64 + +1: ldmia r1!, { r4-r11 } + PLD (r12, #64) + subs r2, r2, #32 + + // NOTE: if r12 is more than 64 ahead of r1, the following ldrhi + // for ARM9 preload will not be safely guarded by the preceding subs. + // When it is safely guarded the only possibility to have SIGSEGV here + // is because the caller overstates the length. + ldrhi r3, [r12], #32 /* cheap ARM9 preload */ + stmia r0!, { r4-r11 } + bhs 1b + + add r2, r2, #32 + + + + +less_than_32_left: + /* + * less than 32 bytes left at this point (length in r2) + */ + + /* skip all this if there is nothing to do, which should + * be a common case (if not executed the code below takes + * about 16 cycles) + */ + tst r2, #0x1F + beq 1f + + /* conditionnaly copies 0 to 31 bytes */ + movs r12, r2, lsl #28 + ldmcsia r1!, {r4, r5, r6, r7} /* 16 bytes */ + ldmmiia r1!, {r8, r9} /* 8 bytes */ + stmcsia r0!, {r4, r5, r6, r7} + stmmiia r0!, {r8, r9} + movs r12, r2, lsl #30 + ldrcs r3, [r1], #4 /* 4 bytes */ + ldrmih r4, [r1], #2 /* 2 bytes */ + strcs r3, [r0], #4 + strmih r4, [r0], #2 + tst r2, #0x1 + ldrneb r3, [r1] /* last byte */ + strneb r3, [r0] + + /* we're done! restore everything and return */ +1: ldmfd sp!, {r5-r11} + ldmfd sp!, {r0, r4, lr} + bx lr + + /********************************************************************/ + +non_congruent: + /* + * here source is aligned to 4 bytes + * but destination is not. + * + * in the code below r2 is the number of bytes read + * (the number of bytes written is always smaller, because we have + * partial words in the shift queue) + */ + cmp r2, #4 + blo copy_last_3_and_return + + /* Use post-incriment mode for stm to spill r5-r11 to reserved stack + * frame. Don't update sp. + */ + stmea sp, {r5-r11} + + /* compute shifts needed to align src to dest */ + rsb r5, r0, #0 + and r5, r5, #3 /* r5 = # bytes in partial words */ + mov r12, r5, lsl #3 /* r12 = right */ + rsb lr, r12, #32 /* lr = left */ + + /* read the first word */ + ldr r3, [r1], #4 + sub r2, r2, #4 + + /* write a partial word (0 to 3 bytes), such that destination + * becomes aligned to 32 bits (r5 = nb of words to copy for alignment) + */ + movs r5, r5, lsl #31 + strmib r3, [r0], #1 + movmi r3, r3, lsr #8 + strcsb r3, [r0], #1 + movcs r3, r3, lsr #8 + strcsb r3, [r0], #1 + movcs r3, r3, lsr #8 + + cmp r2, #4 + blo partial_word_tail + + /* Align destination to 32 bytes (cache line boundary) */ +1: tst r0, #0x1c + beq 2f + ldr r5, [r1], #4 + sub r2, r2, #4 + orr r4, r3, r5, lsl lr + mov r3, r5, lsr r12 + str r4, [r0], #4 + cmp r2, #4 + bhs 1b + blo partial_word_tail + + /* copy 32 bytes at a time */ +2: subs r2, r2, #32 + blo less_than_thirtytwo + + /* Use immediate mode for the shifts, because there is an extra cycle + * for register shifts, which could account for up to 50% of + * performance hit. + */ + + cmp r12, #24 + beq loop24 + cmp r12, #8 + beq loop8 + +loop16: + ldr r12, [r1], #4 +1: mov r4, r12 + ldmia r1!, { r5,r6,r7, r8,r9,r10,r11} + PLD (r1, #64) + subs r2, r2, #32 + ldrhs r12, [r1], #4 + orr r3, r3, r4, lsl #16 + mov r4, r4, lsr #16 + orr r4, r4, r5, lsl #16 + mov r5, r5, lsr #16 + orr r5, r5, r6, lsl #16 + mov r6, r6, lsr #16 + orr r6, r6, r7, lsl #16 + mov r7, r7, lsr #16 + orr r7, r7, r8, lsl #16 + mov r8, r8, lsr #16 + orr r8, r8, r9, lsl #16 + mov r9, r9, lsr #16 + orr r9, r9, r10, lsl #16 + mov r10, r10, lsr #16 + orr r10, r10, r11, lsl #16 + stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10} + mov r3, r11, lsr #16 + bhs 1b + b less_than_thirtytwo + +loop8: + ldr r12, [r1], #4 +1: mov r4, r12 + ldmia r1!, { r5,r6,r7, r8,r9,r10,r11} + PLD (r1, #64) + subs r2, r2, #32 + ldrhs r12, [r1], #4 + orr r3, r3, r4, lsl #24 + mov r4, r4, lsr #8 + orr r4, r4, r5, lsl #24 + mov r5, r5, lsr #8 + orr r5, r5, r6, lsl #24 + mov r6, r6, lsr #8 + orr r6, r6, r7, lsl #24 + mov r7, r7, lsr #8 + orr r7, r7, r8, lsl #24 + mov r8, r8, lsr #8 + orr r8, r8, r9, lsl #24 + mov r9, r9, lsr #8 + orr r9, r9, r10, lsl #24 + mov r10, r10, lsr #8 + orr r10, r10, r11, lsl #24 + stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10} + mov r3, r11, lsr #8 + bhs 1b + b less_than_thirtytwo + +loop24: + ldr r12, [r1], #4 +1: mov r4, r12 + ldmia r1!, { r5,r6,r7, r8,r9,r10,r11} + PLD (r1, #64) + subs r2, r2, #32 + ldrhs r12, [r1], #4 + orr r3, r3, r4, lsl #8 + mov r4, r4, lsr #24 + orr r4, r4, r5, lsl #8 + mov r5, r5, lsr #24 + orr r5, r5, r6, lsl #8 + mov r6, r6, lsr #24 + orr r6, r6, r7, lsl #8 + mov r7, r7, lsr #24 + orr r7, r7, r8, lsl #8 + mov r8, r8, lsr #24 + orr r8, r8, r9, lsl #8 + mov r9, r9, lsr #24 + orr r9, r9, r10, lsl #8 + mov r10, r10, lsr #24 + orr r10, r10, r11, lsl #8 + stmia r0!, {r3,r4,r5,r6, r7,r8,r9,r10} + mov r3, r11, lsr #24 + bhs 1b + + +less_than_thirtytwo: + /* copy the last 0 to 31 bytes of the source */ + rsb r12, lr, #32 /* we corrupted r12, recompute it */ + add r2, r2, #32 + cmp r2, #4 + blo partial_word_tail + +1: ldr r5, [r1], #4 + sub r2, r2, #4 + orr r4, r3, r5, lsl lr + mov r3, r5, lsr r12 + str r4, [r0], #4 + cmp r2, #4 + bhs 1b + +partial_word_tail: + /* we have a partial word in the input buffer */ + movs r5, lr, lsl #(31-3) + strmib r3, [r0], #1 + movmi r3, r3, lsr #8 + strcsb r3, [r0], #1 + movcs r3, r3, lsr #8 + strcsb r3, [r0], #1 + + /* Refill spilled registers from the stack. Don't update sp. */ + ldmfd sp, {r5-r11} + +copy_last_3_and_return: + movs r2, r2, lsl #31 /* copy remaining 0, 1, 2 or 3 bytes */ + ldrmib r2, [r1], #1 + ldrcsb r3, [r1], #1 + ldrcsb r12,[r1] + strmib r2, [r0], #1 + strcsb r3, [r0], #1 + strcsb r12,[r0] + + /* we're done! restore sp and spilled registers and return */ + add sp, sp, #28 + ldmfd sp!, {r0, r4, lr} + bx lr +END(memcpy) diff --git a/libc/arch-arm/generic/bionic/memset.S b/libc/arch-arm/generic/bionic/memset.S new file mode 100644 index 000000000..3c034e015 --- /dev/null +++ b/libc/arch-arm/generic/bionic/memset.S @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include + + /* + * Optimized memset() for ARM. + * + * memset() returns its first argument. + */ + +ENTRY(bzero) + mov r2, r1 + mov r1, #0 +END(bzero) + +ENTRY(memset) + /* compute the offset to align the destination + * offset = (4-(src&3))&3 = -src & 3 + */ + .save {r0, r4-r7, lr} + stmfd sp!, {r0, r4-r7, lr} + rsb r3, r0, #0 + ands r3, r3, #3 + cmp r3, r2 + movhi r3, r2 + + /* splat r1 */ + mov r1, r1, lsl #24 + orr r1, r1, r1, lsr #8 + orr r1, r1, r1, lsr #16 + + movs r12, r3, lsl #31 + strcsb r1, [r0], #1 /* can't use strh (alignment unknown) */ + strcsb r1, [r0], #1 + strmib r1, [r0], #1 + subs r2, r2, r3 + ldmlsfd sp!, {r0, r4-r7, lr} /* return */ + bxls lr + + /* align the destination to a cache-line */ + mov r12, r1 + mov lr, r1 + mov r4, r1 + mov r5, r1 + mov r6, r1 + mov r7, r1 + + rsb r3, r0, #0 + ands r3, r3, #0x1C + beq 3f + cmp r3, r2 + andhi r3, r2, #0x1C + sub r2, r2, r3 + + /* conditionally writes 0 to 7 words (length in r3) */ + movs r3, r3, lsl #28 + stmcsia r0!, {r1, lr} + stmcsia r0!, {r1, lr} + stmmiia r0!, {r1, lr} + movs r3, r3, lsl #2 + strcs r1, [r0], #4 + +3: + subs r2, r2, #32 + mov r3, r1 + bmi 2f +1: subs r2, r2, #32 + stmia r0!, {r1,r3,r4,r5,r6,r7,r12,lr} + bhs 1b +2: add r2, r2, #32 + + /* conditionally stores 0 to 31 bytes */ + movs r2, r2, lsl #28 + stmcsia r0!, {r1,r3,r12,lr} + stmmiia r0!, {r1, lr} + movs r2, r2, lsl #2 + strcs r1, [r0], #4 + strmih r1, [r0], #2 + movs r2, r2, lsl #2 + strcsb r1, [r0] + ldmfd sp!, {r0, r4-r7, lr} + bx lr +END(memset) diff --git a/libc/arch-arm/generic/generic.mk b/libc/arch-arm/generic/generic.mk new file mode 100644 index 000000000..46bd0671d --- /dev/null +++ b/libc/arch-arm/generic/generic.mk @@ -0,0 +1,2 @@ +$(call libc-add-cpu-variant-src,MEMCPY,arch-arm/generic/bionic/memcpy.S) +$(call libc-add-cpu-variant-src,MEMSET,arch-arm/generic/bionic/memset.S) diff --git a/libc/arch-arm/krait/bionic/memcpy.S b/libc/arch-arm/krait/bionic/memcpy.S new file mode 100644 index 000000000..0cd4d445a --- /dev/null +++ b/libc/arch-arm/krait/bionic/memcpy.S @@ -0,0 +1,146 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* Assumes neon instructions and a cache line size of 32 bytes. */ + +#include +#include + +/* + * This code assumes it is running on a processor that supports all arm v7 + * instructions, that supports neon instructions, and that has a 32 byte + * cache line. + */ + + .text + .fpu neon + +#define CACHE_LINE_SIZE 32 + +ENTRY(memcpy) + .save {r0, lr} + /* start preloading as early as possible */ + pld [r1, #(CACHE_LINE_SIZE*0)] + stmfd sp!, {r0, lr} + pld [r1, #(CACHE_LINE_SIZE*2)] + + /* do we have at least 16-bytes to copy (needed for alignment below) */ + cmp r2, #16 + blo 5f + + /* align destination to cache-line for the write-buffer */ + rsb r3, r0, #0 + ands r3, r3, #0xF + beq 0f + + /* copy up to 15-bytes (count in r3) */ + sub r2, r2, r3 + movs ip, r3, lsl #31 + ldrmib lr, [r1], #1 + strmib lr, [r0], #1 + ldrcsb ip, [r1], #1 + ldrcsb lr, [r1], #1 + strcsb ip, [r0], #1 + strcsb lr, [r0], #1 + movs ip, r3, lsl #29 + bge 1f + // copies 4 bytes, destination 32-bits aligned + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]! +1: bcc 2f + // copies 8 bytes, destination 64-bits aligned + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0, :64]! +2: + +0: /* preload immediately the next cache line, which we may need */ + pld [r1, #(CACHE_LINE_SIZE*0)] + pld [r1, #(CACHE_LINE_SIZE*2)] + + /* make sure we have at least 64 bytes to copy */ + subs r2, r2, #64 + blo 2f + + /* Preload all the cache lines we need. + * NOTE: The number of pld below depends on CACHE_LINE_SIZE, + * ideally we would increase the distance in the main loop to + * avoid the goofy code below. In practice this doesn't seem to make + * a big difference. + * NOTE: The value CACHE_LINE_SIZE * 8 was chosen through + * experimentation. + */ + pld [r1, #(CACHE_LINE_SIZE*4)] + pld [r1, #(CACHE_LINE_SIZE*6)] + pld [r1, #(CACHE_LINE_SIZE*8)] + +1: /* The main loop copies 64 bytes at a time */ + vld1.8 {d0 - d3}, [r1]! + vld1.8 {d4 - d7}, [r1]! + pld [r1, #(CACHE_LINE_SIZE*8)] + subs r2, r2, #64 + vst1.8 {d0 - d3}, [r0, :128]! + vst1.8 {d4 - d7}, [r0, :128]! + bhs 1b + +2: /* fix-up the remaining count and make sure we have >= 32 bytes left */ + add r2, r2, #64 + subs r2, r2, #32 + blo 4f + +3: /* 32 bytes at a time. These cache lines were already preloaded */ + vld1.8 {d0 - d3}, [r1]! + subs r2, r2, #32 + vst1.8 {d0 - d3}, [r0, :128]! + bhs 3b +4: /* less than 32 left */ + add r2, r2, #32 + tst r2, #0x10 + beq 5f + // copies 16 bytes, 128-bits aligned + vld1.8 {d0, d1}, [r1]! + vst1.8 {d0, d1}, [r0, :128]! + +5: /* copy up to 15-bytes (count in r2) */ + movs ip, r2, lsl #29 + bcc 1f + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0]! +1: bge 2f + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]! +2: movs ip, r2, lsl #31 + ldrmib r3, [r1], #1 + ldrcsb ip, [r1], #1 + ldrcsb lr, [r1], #1 + strmib r3, [r0], #1 + strcsb ip, [r0], #1 + strcsb lr, [r0], #1 + + ldmfd sp!, {r0, lr} + bx lr +END(memcpy) diff --git a/libc/arch-arm/krait/bionic/memset.S b/libc/arch-arm/krait/bionic/memset.S new file mode 100644 index 000000000..a2e2d80dc --- /dev/null +++ b/libc/arch-arm/krait/bionic/memset.S @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include + +/* + * This code assumes it is running on a processor that supports all arm v7 + * instructions, that supports neon instructions, and that supports + * unaligned neon instruction accesses to memory. + */ + + .fpu neon + +ENTRY(bzero) + mov r2, r1 + mov r1, #0 +END(bzero) + +/* memset() returns its first argument. */ +ENTRY(memset) + .save {r0} + stmfd sp!, {r0} + + vdup.8 q0, r1 + + /* make sure we have at least 32 bytes to write */ + subs r2, r2, #32 + blo 2f + vmov q1, q0 + +1: /* The main loop writes 32 bytes at a time */ + subs r2, r2, #32 + vst1.8 {d0 - d3}, [r0]! + bhs 1b + +2: /* less than 32 left */ + add r2, r2, #32 + tst r2, #0x10 + beq 3f + + // writes 16 bytes, 128-bits aligned + vst1.8 {d0, d1}, [r0]! +3: /* write up to 15-bytes (count in r2) */ + movs ip, r2, lsl #29 + bcc 1f + vst1.8 {d0}, [r0]! +1: bge 2f + vst1.32 {d0[0]}, [r0]! +2: movs ip, r2, lsl #31 + strmib r1, [r0], #1 + strcsb r1, [r0], #1 + strcsb r1, [r0], #1 + ldmfd sp!, {r0} + bx lr +END(memset) diff --git a/libc/arch-arm/krait/krait.mk b/libc/arch-arm/krait/krait.mk new file mode 100644 index 000000000..ab55d2403 --- /dev/null +++ b/libc/arch-arm/krait/krait.mk @@ -0,0 +1,4 @@ +$(call libc-add-cpu-variant-src,MEMCPY,arch-arm/krait/bionic/memcpy.S) +$(call libc-add-cpu-variant-src,MEMSET,arch-arm/krait/bionic/memset.S) + +include bionic/libc/arch-arm/generic/generic.mk diff --git a/libc/arch-mips/mips.mk b/libc/arch-mips/mips.mk new file mode 100644 index 000000000..fe57ee791 --- /dev/null +++ b/libc/arch-mips/mips.mk @@ -0,0 +1,25 @@ +_LIBC_ARCH_COMMON_SRC_FILES := \ + arch-mips/bionic/__get_sp.S \ + arch-mips/bionic/__get_tls.c \ + arch-mips/bionic/__set_tls.c \ + arch-mips/bionic/_exit_with_stack_teardown.S \ + arch-mips/bionic/_setjmp.S \ + arch-mips/bionic/futex_mips.S \ + arch-mips/bionic/bzero.S \ + arch-mips/bionic/cacheflush.c \ + arch-mips/bionic/clone.S \ + arch-mips/bionic/ffs.S \ + arch-mips/bionic/memcmp16.S \ + arch-mips/bionic/memmove.c \ + arch-mips/bionic/pipe.S \ + arch-mips/bionic/setjmp.S \ + arch-mips/bionic/sigsetjmp.S \ + arch-mips/bionic/vfork.S \ + arch-mips/string/memset.S \ + arch-mips/string/memcpy.S \ + arch-mips/string/mips_strlen.c \ + +_LIBC_ARCH_STATIC_SRC_FILES := \ + bionic/dl_iterate_phdr_static.c \ + +_LIBC_ARCH_DYNAMIC_SRC_FILES := diff --git a/libc/arch-x86/x86.mk b/libc/arch-x86/x86.mk new file mode 100644 index 000000000..0e5d28300 --- /dev/null +++ b/libc/arch-x86/x86.mk @@ -0,0 +1,27 @@ +_LIBC_ARCH_COMMON_SRC_FILES := \ + arch-x86/bionic/clone.S \ + arch-x86/bionic/_exit_with_stack_teardown.S \ + arch-x86/bionic/futex_x86.S \ + arch-x86/bionic/__get_sp.S \ + arch-x86/bionic/__get_tls.c \ + arch-x86/bionic/_setjmp.S \ + arch-x86/bionic/setjmp.S \ + arch-x86/bionic/__set_tls.c \ + arch-x86/bionic/sigsetjmp.S \ + arch-x86/bionic/syscall.S \ + arch-x86/bionic/vfork.S \ + arch-x86/string/bcopy_wrapper.S \ + arch-x86/string/bzero_wrapper.S \ + arch-x86/string/ffs.S \ + arch-x86/string/memcmp_wrapper.S \ + arch-x86/string/memcpy_wrapper.S \ + arch-x86/string/memmove_wrapper.S \ + arch-x86/string/memset_wrapper.S \ + arch-x86/string/strcmp_wrapper.S \ + arch-x86/string/strlen_wrapper.S \ + arch-x86/string/strncmp_wrapper.S \ + +_LIBC_ARCH_STATIC_SRC_FILES := \ + bionic/dl_iterate_phdr_static.c \ + +_LIBC_ARCH_DYNAMIC_SRC_FILES := From 6e1a5cf31ba47508b08dba02a45b4ea6e1edd6d2 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Wed, 27 Feb 2013 16:25:03 -0800 Subject: [PATCH 02/77] Remove unused arm defines. The defines HAVE_32_BYTE_CACHE_LINES and ARCH_ARM_USE_NON_NEON_MEMCPY are not used by any code. The previous memcpy code that used these has been split into different architecture versions to avoid the need for them. Bug: 8005082 Change-Id: I14e0368c5bb4c3a80e72520f7cfd97e359918cce --- libc/Android.mk | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/libc/Android.mk b/libc/Android.mk index 9a55eff27..2d0c3510c 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -491,17 +491,6 @@ ifeq ($(TARGET_ARCH),arm) libc_common_cflags += -DSOFTFLOAT libc_common_cflags += -fstrict-aliasing libc_crt_target_cflags := -mthumb-interwork - # - # Define HAVE_32_BYTE_CACHE_LINES to indicate to the C library - # that it should use the 32-byte version of memcpy, not - # the 64-byte version. - # - ifeq ($(ARCH_ARM_HAVE_32_BYTE_CACHE_LINES),true) - libc_common_cflags += -DHAVE_32_BYTE_CACHE_LINE - endif - ifeq ($(ARCH_ARM_USE_NON_NEON_MEMCPY),true) - libc_common_cflags += -DARCH_ARM_USE_NON_NEON_MEMCPY - endif endif # !arm ifeq ($(TARGET_ARCH),x86) From fbae9f3c30c869ca2fe8609f2d680430f543f0e6 Mon Sep 17 00:00:00 2001 From: Sasha Levitskiy Date: Wed, 27 Feb 2013 15:48:55 -0800 Subject: [PATCH 03/77] Bionic: Libc: Resolv: Stricter function signatures, unitialized return bug fix. Stricter input parameters help avoid ugly casting when passing pointers to immutable protobuf data. While at it: an int return was dropped from 2 functions whose users never used the result; one of the return paths was returning an uninitialized value. Size_t for portablity and warning supression, misc warnings addressed. Change-Id: I2d5cbdaf0c9b6c4621a7d397772da13da5dc0943 --- libc/netbsd/resolv/res_cache.c | 66 +++++++++++++++++----------------- libc/private/resolv_cache.h | 11 +++--- libc/private/resolv_iface.h | 2 +- 3 files changed, 41 insertions(+), 38 deletions(-) diff --git a/libc/netbsd/resolv/res_cache.c b/libc/netbsd/resolv/res_cache.c index 08a25576c..8e06b1f21 100644 --- a/libc/netbsd/resolv/res_cache.c +++ b/libc/netbsd/resolv/res_cache.c @@ -1834,7 +1834,7 @@ static struct in_addr* _get_addr_locked(const char * ifname); /* return 1 if the provided list of name servers differs from the list of name servers * currently attached to the provided cache_info */ static int _resolv_is_nameservers_equal_locked(struct resolv_cache_info* cache_info, - char** servers, int numservers); + const char** servers, int numservers); /* remove a resolv_pidiface_info structure from _res_pidiface_list */ static void _remove_pidiface_info_locked(int pid); /* get a resolv_pidiface_info structure from _res_pidiface_list with a certain pid */ @@ -2077,7 +2077,7 @@ _resolv_set_default_iface(const char* ifname) } void -_resolv_set_nameservers_for_iface(const char* ifname, char** servers, int numservers, +_resolv_set_nameservers_for_iface(const char* ifname, const char** servers, int numservers, const char *domains) { int i, rt, index; @@ -2150,7 +2150,7 @@ _resolv_set_nameservers_for_iface(const char* ifname, char** servers, int numser static int _resolv_is_nameservers_equal_locked(struct resolv_cache_info* cache_info, - char** servers, int numservers) + const char** servers, int numservers) { int i; char** ns; @@ -2272,8 +2272,8 @@ _resolv_set_addr_of_iface(const char* ifname, struct in_addr* addr) memcpy(&cache_info->ifaddr, addr, sizeof(*addr)); if (DEBUG) { - char* addr_s = inet_ntoa(cache_info->ifaddr); - XLOG("address of interface %s is %s\n", ifname, addr_s); + XLOG("address of interface %s is %s\n", + ifname, inet_ntoa(cache_info->ifaddr)); } } pthread_mutex_unlock(&_res_cache_list_lock); @@ -2412,26 +2412,24 @@ _resolv_get_pids_associated_interface(int pid, char* buff, int buffLen) return len; } -int -_resolv_get_default_iface(char* buff, int buffLen) +size_t +_resolv_get_default_iface(char* buff, size_t buffLen) { - char* ifname; - int len = 0; - if (!buff || buffLen == 0) { - return -1; + return 0; } pthread_once(&_res_cache_once, _res_cache_init); pthread_mutex_lock(&_res_cache_list_lock); - ifname = _get_default_iface_locked(); // never null, but may be empty + char* ifname = _get_default_iface_locked(); // never null, but may be empty // if default interface not set. Get first cache with an interface if (ifname[0] == '\0') { ifname = _find_any_iface_name_locked(); // may be null } + size_t len = 0; // if we got the default iface or if (no-default) the find_any call gave an answer if (ifname) { len = strlen(ifname); @@ -2448,28 +2446,32 @@ _resolv_get_default_iface(char* buff, int buffLen) return len; } -int +void _resolv_populate_res_for_iface(res_state statp) { - int nserv; - struct resolv_cache_info* info = NULL; + if (statp == NULL) { + return; + } - if (statp) { + if (statp->iface[0] == '\0') { // no interface set assign default + size_t if_len = _resolv_get_default_iface(statp->iface, sizeof(statp->iface)); + if (if_len + 1 > sizeof(statp->iface)) { + XLOG("%s: INTERNAL_ERROR: can't fit interface name into statp->iface.\n", __FUNCTION__); + return; + } + if (if_len == 0) { + XLOG("%s: INTERNAL_ERROR: can't find any suitable interfaces.\n", __FUNCTION__); + return; + } + } + + pthread_once(&_res_cache_once, _res_cache_init); + pthread_mutex_lock(&_res_cache_list_lock); + + struct resolv_cache_info* info = _find_cache_info_locked(statp->iface); + if (info != NULL) { + int nserv; struct addrinfo* ai; - - if (statp->iface[0] == '\0') { // no interface set assign default - _resolv_get_default_iface(statp->iface, sizeof(statp->iface)); - } - - pthread_once(&_res_cache_once, _res_cache_init); - pthread_mutex_lock(&_res_cache_list_lock); - info = _find_cache_info_locked(statp->iface); - - if (info == NULL) { - pthread_mutex_unlock(&_res_cache_list_lock); - return 0; - } - XLOG("_resolv_populate_res_for_iface: %s\n", statp->iface); for (nserv = 0; nserv < MAXNS; nserv++) { ai = info->nsaddrinfo[nserv]; @@ -2503,8 +2505,6 @@ _resolv_populate_res_for_iface(res_state statp) while (pp < statp->dnsrch + MAXDNSRCH && *p != -1) { *pp++ = &statp->defdname + *p++; } - - pthread_mutex_unlock(&_res_cache_list_lock); } - return nserv; + pthread_mutex_unlock(&_res_cache_list_lock); } diff --git a/libc/private/resolv_cache.h b/libc/private/resolv_cache.h index 4f32fb75f..bf9124e33 100644 --- a/libc/private/resolv_cache.h +++ b/libc/private/resolv_cache.h @@ -28,6 +28,8 @@ #ifndef _RESOLV_CACHE_H_ #define _RESOLV_CACHE_H_ +#include + struct __res_state; struct resolv_cache; /* forward */ @@ -67,14 +69,15 @@ extern struct in_addr* _resolv_get_addr_of_default_iface(); /* gets the address associated with the specified interface */ extern struct in_addr* _resolv_get_addr_of_iface(const char* ifname); -/* Copy the name of the default interface to provided buffer. - * Return length of buffer on success on failure -1 is returned */ -extern int _resolv_get_default_iface(char* buff, int buffLen); +/* Copy the name of the default interface to the provided buffer. + * Returns the string length of the default interface, + * be that less or more than the buffLen, or 0 if nothing had been written */ + extern size_t _resolv_get_default_iface(char* buff, size_t buffLen); /* sets the name server addresses to the provided res_state structure. The * name servers are retrieved from the cache which is associated * with the interface to which the res_state structure is associated */ -extern int _resolv_populate_res_for_iface(struct __res_state* statp); +extern void _resolv_populate_res_for_iface(struct __res_state* statp); typedef enum { RESOLV_CACHE_UNSUPPORTED, /* the cache can't handle that kind of queries */ diff --git a/libc/private/resolv_iface.h b/libc/private/resolv_iface.h index bf5abad7d..789d99719 100644 --- a/libc/private/resolv_iface.h +++ b/libc/private/resolv_iface.h @@ -48,7 +48,7 @@ __BEGIN_DECLS extern void _resolv_set_default_iface(const char* ifname); /* set name servers for an interface */ -extern void _resolv_set_nameservers_for_iface(const char* ifname, char** servers, int numservers, +extern void _resolv_set_nameservers_for_iface(const char* ifname, const char** servers, int numservers, const char *domains); /* tell resolver of the address of an interface */ From a9a5870d166f8060a8182cd61e5536b0becea74e Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Fri, 8 Mar 2013 16:50:31 -0800 Subject: [PATCH 04/77] Create arch specific versions of strcmp. This uses the new strcmp.a15.S code as the basis for new versions of strcmp.S. The cortex-a15 code is the performance optimized version of strcmp.a15.S taken with only the addition of a few pld instructions. The cortex-a9 code is the same as the cortex-a15 code except that the unaligned strcmp code was taken from the original strcmp.S. The krait code is the same as the cortex-a15 code except that one path in the unaligned strcmp code was taken from the original strcmp.S code (the 2 byte overlap case). The generic code is the original unmodified strmp.S from the bionic subdirectory. All three new versions underwent these test cases: Strings the same, all same size: - Both pointers double word aligned. - One pointer double word aligned, one pointer word aligned. - Both pointers word aligned. - One pointer double word aligned, one pointer 1 off a word alignment. - One pointer double word aligned, one pointer 2 off a word alignment. - One pointer double word aligned, one pointer 3 off a word alignment. - One pointer word aligned, one pointer 1 off a word alignment. - One pointer word aligned, one pointer 2 off a word alignment. - One pointer word aligned, one pointer 3 off a word alignment. For all cases where it made sense, the two pointers were also tested swapped. Different strings, all same size: - Single difference at double word boundary. - Single difference at word boudary. - Single difference at 1 off a word alignment. - Single difference at 2 off a word alignment. - Single difference at 3 off a word alignment. Different sized strings, strings the same until the end: - Shorter string ends on a double word boundary. - Shorter string ends on word boundary. - Shorter string ends at 1 off a word boundary. - Shorter string ends at 2 off a word boundary. - Shorter string ends at 3 off a word boundary. For all different cases, run them through the same pointer alignment cases when the strings are the same size. For all cases the two pointers were also tested swapped. Bug: 8005082 Change-Id: I5f3dc02b48afba2cb9c13332ab45c828ff171a1c --- libc/arch-arm/arm.mk | 1 - libc/arch-arm/cortex-a15/bionic/strcmp.S | 377 ++++++++++++++++ libc/arch-arm/cortex-a15/cortex-a15.mk | 1 + libc/arch-arm/cortex-a9/bionic/strcmp.S | 544 +++++++++++++++++++++++ libc/arch-arm/cortex-a9/cortex-a9.mk | 1 + libc/arch-arm/generic/bionic/strcmp.S | 317 +++++++++++++ libc/arch-arm/generic/generic.mk | 1 + libc/arch-arm/krait/bionic/strcmp.S | 477 ++++++++++++++++++++ libc/arch-arm/krait/krait.mk | 1 + 9 files changed, 1719 insertions(+), 1 deletion(-) create mode 100644 libc/arch-arm/cortex-a15/bionic/strcmp.S create mode 100644 libc/arch-arm/cortex-a9/bionic/strcmp.S create mode 100644 libc/arch-arm/generic/bionic/strcmp.S create mode 100644 libc/arch-arm/krait/bionic/strcmp.S diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index 2def30b17..d15ec9d1b 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -14,7 +14,6 @@ _LIBC_ARCH_COMMON_SRC_FILES := \ arch-arm/bionic/_setjmp.S \ arch-arm/bionic/setjmp.S \ arch-arm/bionic/sigsetjmp.S \ - arch-arm/bionic/strcmp.S \ arch-arm/bionic/strcpy.S \ arch-arm/bionic/strlen.c.arm \ arch-arm/bionic/syscall.S \ diff --git a/libc/arch-arm/cortex-a15/bionic/strcmp.S b/libc/arch-arm/cortex-a15/bionic/strcmp.S new file mode 100644 index 000000000..7aff7c4e1 --- /dev/null +++ b/libc/arch-arm/cortex-a15/bionic/strcmp.S @@ -0,0 +1,377 @@ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#ifdef __ARMEB__ +#define S2LOMEM lsl +#define S2LOMEMEQ lsleq +#define S2HIMEM lsr +#define MSB 0x000000ff +#define LSB 0xff000000 +#define BYTE0_OFFSET 24 +#define BYTE1_OFFSET 16 +#define BYTE2_OFFSET 8 +#define BYTE3_OFFSET 0 +#else /* not __ARMEB__ */ +#define S2LOMEM lsr +#define S2LOMEMEQ lsreq +#define S2HIMEM lsl +#define BYTE0_OFFSET 0 +#define BYTE1_OFFSET 8 +#define BYTE2_OFFSET 16 +#define BYTE3_OFFSET 24 +#define MSB 0xff000000 +#define LSB 0x000000ff +#endif /* not __ARMEB__ */ + +.syntax unified + +#if defined (__thumb__) + .thumb + .thumb_func +#endif + +ENTRY(strcmp) + /* Use LDRD whenever possible. */ + +/* The main thing to look out for when comparing large blocks is that + the loads do not cross a page boundary when loading past the index + of the byte with the first difference or the first string-terminator. + + For example, if the strings are identical and the string-terminator + is at index k, byte by byte comparison will not load beyond address + s1+k and s2+k; word by word comparison may load up to 3 bytes beyond + k; double word - up to 7 bytes. If the load of these bytes crosses + a page boundary, it might cause a memory fault (if the page is not mapped) + that would not have happened in byte by byte comparison. + + If an address is (double) word aligned, then a load of a (double) word + from that address will not cross a page boundary. + Therefore, the algorithm below considers word and double-word alignment + of strings separately. */ + +/* High-level description of the algorithm. + + * The fast path: if both strings are double-word aligned, + use LDRD to load two words from each string in every loop iteration. + * If the strings have the same offset from a word boundary, + use LDRB to load and compare byte by byte until + the first string is aligned to a word boundary (at most 3 bytes). + This is optimized for quick return on short unaligned strings. + * If the strings have the same offset from a double-word boundary, + use LDRD to load two words from each string in every loop iteration, as in the fast path. + * If the strings do not have the same offset from a double-word boundary, + load a word from the second string before the loop to initialize the queue. + Use LDRD to load two words from every string in every loop iteration. + Inside the loop, load the second word from the second string only after comparing + the first word, using the queued value, to guarantee safety across page boundaries. + * If the strings do not have the same offset from a word boundary, + use LDR and a shift queue. Order of loads and comparisons matters, + similarly to the previous case. + + * Use UADD8 and SEL to compare words, and use REV and CLZ to compute the return value. + * The only difference between ARM and Thumb modes is the use of CBZ instruction. + * The only difference between big and little endian is the use of REV in little endian + to compute the return value, instead of MOV. +*/ + + .macro m_cbz reg label +#ifdef __thumb2__ + cbz \reg, \label +#else /* not defined __thumb2__ */ + cmp \reg, #0 + beq \label +#endif /* not defined __thumb2__ */ + .endm /* m_cbz */ + + .macro m_cbnz reg label +#ifdef __thumb2__ + cbnz \reg, \label +#else /* not defined __thumb2__ */ + cmp \reg, #0 + bne \label +#endif /* not defined __thumb2__ */ + .endm /* m_cbnz */ + + .macro init + /* Macro to save temporary registers and prepare magic values. */ + subs sp, sp, #16 + strd r4, r5, [sp, #8] + strd r6, r7, [sp] + mvn r6, #0 /* all F */ + mov r7, #0 /* all 0 */ + .endm /* init */ + + .macro magic_compare_and_branch w1 w2 label + /* Macro to compare registers w1 and w2 and conditionally branch to label. */ + cmp \w1, \w2 /* Are w1 and w2 the same? */ + magic_find_zero_bytes \w1 + it eq + cmpeq ip, #0 /* Is there a zero byte in w1? */ + bne \label + .endm /* magic_compare_and_branch */ + + .macro magic_find_zero_bytes w1 + /* Macro to find all-zero bytes in w1, result is in ip. */ +#if (defined (__ARM_FEATURE_DSP)) + uadd8 ip, \w1, r6 + sel ip, r7, r6 +#else /* not defined (__ARM_FEATURE_DSP) */ + /* __ARM_FEATURE_DSP is not defined for some Cortex-M processors. + Coincidently, these processors only have Thumb-2 mode, where we can use the + the (large) magic constant available directly as an immediate in instructions. + Note that we cannot use the magic constant in ARM mode, where we need + to create the constant in a register. */ + sub ip, \w1, #0x01010101 + bic ip, ip, \w1 + and ip, ip, #0x80808080 +#endif /* not defined (__ARM_FEATURE_DSP) */ + .endm /* magic_find_zero_bytes */ + + .macro setup_return w1 w2 +#ifdef __ARMEB__ + mov r1, \w1 + mov r2, \w2 +#else /* not __ARMEB__ */ + rev r1, \w1 + rev r2, \w2 +#endif /* not __ARMEB__ */ + .endm /* setup_return */ + + pld [r0, #0] + pld [r1, #0] + + /* Are both strings double-word aligned? */ + orr ip, r0, r1 + tst ip, #7 + bne do_align + + /* Fast path. */ + init + +doubleword_aligned: + + /* Get here when the strings to compare are double-word aligned. */ + /* Compare two words in every iteration. */ + .p2align 2 +2: + pld [r0, #16] + pld [r1, #16] + + /* Load the next double-word from each string. */ + ldrd r2, r3, [r0], #8 + ldrd r4, r5, [r1], #8 + + magic_compare_and_branch w1=r2, w2=r4, label=return_24 + magic_compare_and_branch w1=r3, w2=r5, label=return_35 + b 2b + +do_align: + /* Is the first string word-aligned? */ + ands ip, r0, #3 + beq word_aligned_r0 + + /* Fast compare byte by byte until the first string is word-aligned. */ + /* The offset of r0 from a word boundary is in ip. Thus, the number of bytes + to read until the next word boundary is 4-ip. */ + bic r0, r0, #3 + ldr r2, [r0], #4 + lsls ip, ip, #31 + beq byte2 + bcs byte3 + +byte1: + ldrb ip, [r1], #1 + uxtb r3, r2, ror #BYTE1_OFFSET + subs ip, r3, ip + bne fast_return + m_cbz reg=r3, label=fast_return + +byte2: + ldrb ip, [r1], #1 + uxtb r3, r2, ror #BYTE2_OFFSET + subs ip, r3, ip + bne fast_return + m_cbz reg=r3, label=fast_return + +byte3: + ldrb ip, [r1], #1 + uxtb r3, r2, ror #BYTE3_OFFSET + subs ip, r3, ip + bne fast_return + m_cbnz reg=r3, label=word_aligned_r0 + +fast_return: + mov r0, ip + bx lr + +word_aligned_r0: + init + /* The first string is word-aligned. */ + /* Is the second string word-aligned? */ + ands ip, r1, #3 + bne strcmp_unaligned + +word_aligned: + /* The strings are word-aligned. */ + /* Is the first string double-word aligned? */ + tst r0, #4 + beq doubleword_aligned_r0 + + /* If r0 is not double-word aligned yet, align it by loading + and comparing the next word from each string. */ + ldr r2, [r0], #4 + ldr r4, [r1], #4 + magic_compare_and_branch w1=r2 w2=r4 label=return_24 + +doubleword_aligned_r0: + /* Get here when r0 is double-word aligned. */ + /* Is r1 doubleword_aligned? */ + tst r1, #4 + beq doubleword_aligned + + /* Get here when the strings to compare are word-aligned, + r0 is double-word aligned, but r1 is not double-word aligned. */ + + /* Initialize the queue. */ + ldr r5, [r1], #4 + + /* Compare two words in every iteration. */ + .p2align 2 +3: + pld [r0, #16] + pld [r1, #16] + + /* Load the next double-word from each string and compare. */ + ldrd r2, r3, [r0], #8 + magic_compare_and_branch w1=r2 w2=r5 label=return_25 + ldrd r4, r5, [r1], #8 + magic_compare_and_branch w1=r3 w2=r4 label=return_34 + b 3b + + .macro miscmp_word offsetlo offsethi + /* Macro to compare misaligned strings. */ + /* r0, r1 are word-aligned, and at least one of the strings + is not double-word aligned. */ + /* Compare one word in every loop iteration. */ + /* OFFSETLO is the original bit-offset of r1 from a word-boundary, + OFFSETHI is 32 - OFFSETLO (i.e., offset from the next word). */ + + /* Initialize the shift queue. */ + ldr r5, [r1], #4 + + /* Compare one word from each string in every loop iteration. */ + .p2align 2 +7: + ldr r3, [r0], #4 + S2LOMEM r5, r5, #\offsetlo + magic_find_zero_bytes w1=r3 + cmp r7, ip, S2HIMEM #\offsetlo + and r2, r3, r6, S2LOMEM #\offsetlo + it eq + cmpeq r2, r5 + bne return_25 + ldr r5, [r1], #4 + cmp ip, #0 + eor r3, r2, r3 + S2HIMEM r2, r5, #\offsethi + it eq + cmpeq r3, r2 + bne return_32 + b 7b + .endm /* miscmp_word */ + +strcmp_unaligned: + /* r0 is word-aligned, r1 is at offset ip from a word. */ + /* Align r1 to the (previous) word-boundary. */ + bic r1, r1, #3 + + /* Unaligned comparison word by word using LDRs. */ + cmp ip, #2 + beq miscmp_word_16 /* If ip == 2. */ + bge miscmp_word_24 /* If ip == 3. */ + miscmp_word offsetlo=8 offsethi=24 /* If ip == 1. */ +miscmp_word_16: miscmp_word offsetlo=16 offsethi=16 +miscmp_word_24: miscmp_word offsetlo=24 offsethi=8 + + +return_32: + setup_return w1=r3, w2=r2 + b do_return +return_34: + setup_return w1=r3, w2=r4 + b do_return +return_25: + setup_return w1=r2, w2=r5 + b do_return +return_35: + setup_return w1=r3, w2=r5 + b do_return +return_24: + setup_return w1=r2, w2=r4 + +do_return: + +#ifdef __ARMEB__ + mov r0, ip +#else /* not __ARMEB__ */ + rev r0, ip +#endif /* not __ARMEB__ */ + + /* Restore temporaries early, before computing the return value. */ + ldrd r6, r7, [sp] + ldrd r4, r5, [sp, #8] + adds sp, sp, #16 + + /* There is a zero or a different byte between r1 and r2. */ + /* r0 contains a mask of all-zero bytes in r1. */ + /* Using r0 and not ip here because cbz requires low register. */ + m_cbz reg=r0, label=compute_return_value + clz r0, r0 + /* r0 contains the number of bits on the left of the first all-zero byte in r1. */ + rsb r0, r0, #24 + /* Here, r0 contains the number of bits on the right of the first all-zero byte in r1. */ + lsr r1, r1, r0 + lsr r2, r2, r0 + +compute_return_value: + movs r0, #1 + cmp r1, r2 + /* The return value is computed as follows. + If r1>r2 then (C==1 and Z==0) and LS doesn't hold and r0 is #1 at return. + If r1 +#include + +#ifdef __ARMEB__ +#define S2LOMEM lsl +#define S2LOMEMEQ lsleq +#define S2HIMEM lsr +#define MSB 0x000000ff +#define LSB 0xff000000 +#define BYTE0_OFFSET 24 +#define BYTE1_OFFSET 16 +#define BYTE2_OFFSET 8 +#define BYTE3_OFFSET 0 +#else /* not __ARMEB__ */ +#define S2LOMEM lsr +#define S2LOMEMEQ lsreq +#define S2HIMEM lsl +#define BYTE0_OFFSET 0 +#define BYTE1_OFFSET 8 +#define BYTE2_OFFSET 16 +#define BYTE3_OFFSET 24 +#define MSB 0xff000000 +#define LSB 0x000000ff +#endif /* not __ARMEB__ */ + +.syntax unified + +#if defined (__thumb__) + .thumb + .thumb_func +#endif + +ENTRY(strcmp) + /* Use LDRD whenever possible. */ + +/* The main thing to look out for when comparing large blocks is that + the loads do not cross a page boundary when loading past the index + of the byte with the first difference or the first string-terminator. + + For example, if the strings are identical and the string-terminator + is at index k, byte by byte comparison will not load beyond address + s1+k and s2+k; word by word comparison may load up to 3 bytes beyond + k; double word - up to 7 bytes. If the load of these bytes crosses + a page boundary, it might cause a memory fault (if the page is not mapped) + that would not have happened in byte by byte comparison. + + If an address is (double) word aligned, then a load of a (double) word + from that address will not cross a page boundary. + Therefore, the algorithm below considers word and double-word alignment + of strings separately. */ + +/* High-level description of the algorithm. + + * The fast path: if both strings are double-word aligned, + use LDRD to load two words from each string in every loop iteration. + * If the strings have the same offset from a word boundary, + use LDRB to load and compare byte by byte until + the first string is aligned to a word boundary (at most 3 bytes). + This is optimized for quick return on short unaligned strings. + * If the strings have the same offset from a double-word boundary, + use LDRD to load two words from each string in every loop iteration, as in the fast path. + * If the strings do not have the same offset from a double-word boundary, + load a word from the second string before the loop to initialize the queue. + Use LDRD to load two words from every string in every loop iteration. + Inside the loop, load the second word from the second string only after comparing + the first word, using the queued value, to guarantee safety across page boundaries. + * If the strings do not have the same offset from a word boundary, + use LDR and a shift queue. Order of loads and comparisons matters, + similarly to the previous case. + + * Use UADD8 and SEL to compare words, and use REV and CLZ to compute the return value. + * The only difference between ARM and Thumb modes is the use of CBZ instruction. + * The only difference between big and little endian is the use of REV in little endian + to compute the return value, instead of MOV. +*/ + + .macro m_cbz reg label +#ifdef __thumb2__ + cbz \reg, \label +#else /* not defined __thumb2__ */ + cmp \reg, #0 + beq \label +#endif /* not defined __thumb2__ */ + .endm /* m_cbz */ + + .macro m_cbnz reg label +#ifdef __thumb2__ + cbnz \reg, \label +#else /* not defined __thumb2__ */ + cmp \reg, #0 + bne \label +#endif /* not defined __thumb2__ */ + .endm /* m_cbnz */ + + .macro init + /* Macro to save temporary registers and prepare magic values. */ + subs sp, sp, #16 + strd r4, r5, [sp, #8] + strd r6, r7, [sp] + mvn r6, #0 /* all F */ + mov r7, #0 /* all 0 */ + .endm /* init */ + + .macro magic_compare_and_branch w1 w2 label + /* Macro to compare registers w1 and w2 and conditionally branch to label. */ + cmp \w1, \w2 /* Are w1 and w2 the same? */ + magic_find_zero_bytes \w1 + it eq + cmpeq ip, #0 /* Is there a zero byte in w1? */ + bne \label + .endm /* magic_compare_and_branch */ + + .macro magic_find_zero_bytes w1 + /* Macro to find all-zero bytes in w1, result is in ip. */ +#if (defined (__ARM_FEATURE_DSP)) + uadd8 ip, \w1, r6 + sel ip, r7, r6 +#else /* not defined (__ARM_FEATURE_DSP) */ + /* __ARM_FEATURE_DSP is not defined for some Cortex-M processors. + Coincidently, these processors only have Thumb-2 mode, where we can use the + the (large) magic constant available directly as an immediate in instructions. + Note that we cannot use the magic constant in ARM mode, where we need + to create the constant in a register. */ + sub ip, \w1, #0x01010101 + bic ip, ip, \w1 + and ip, ip, #0x80808080 +#endif /* not defined (__ARM_FEATURE_DSP) */ + .endm /* magic_find_zero_bytes */ + + .macro setup_return w1 w2 +#ifdef __ARMEB__ + mov r1, \w1 + mov r2, \w2 +#else /* not __ARMEB__ */ + rev r1, \w1 + rev r2, \w2 +#endif /* not __ARMEB__ */ + .endm /* setup_return */ + + pld [r0, #0] + pld [r1, #0] + + /* Are both strings double-word aligned? */ + orr ip, r0, r1 + tst ip, #7 + bne do_align + + /* Fast path. */ + init + +doubleword_aligned: + + /* Get here when the strings to compare are double-word aligned. */ + /* Compare two words in every iteration. */ + .p2align 2 +2: + pld [r0, #16] + pld [r1, #16] + + /* Load the next double-word from each string. */ + ldrd r2, r3, [r0], #8 + ldrd r4, r5, [r1], #8 + + magic_compare_and_branch w1=r2, w2=r4, label=return_24 + magic_compare_and_branch w1=r3, w2=r5, label=return_35 + b 2b + +do_align: + /* Is the first string word-aligned? */ + ands ip, r0, #3 + beq word_aligned_r0 + + /* Fast compare byte by byte until the first string is word-aligned. */ + /* The offset of r0 from a word boundary is in ip. Thus, the number of bytes + to read until the next word boundary is 4-ip. */ + bic r0, r0, #3 + ldr r2, [r0], #4 + lsls ip, ip, #31 + beq byte2 + bcs byte3 + +byte1: + ldrb ip, [r1], #1 + uxtb r3, r2, ror #BYTE1_OFFSET + subs ip, r3, ip + bne fast_return + m_cbz reg=r3, label=fast_return + +byte2: + ldrb ip, [r1], #1 + uxtb r3, r2, ror #BYTE2_OFFSET + subs ip, r3, ip + bne fast_return + m_cbz reg=r3, label=fast_return + +byte3: + ldrb ip, [r1], #1 + uxtb r3, r2, ror #BYTE3_OFFSET + subs ip, r3, ip + bne fast_return + m_cbnz reg=r3, label=word_aligned_r0 + +fast_return: + mov r0, ip + bx lr + +word_aligned_r0: + init + /* The first string is word-aligned. */ + /* Is the second string word-aligned? */ + ands ip, r1, #3 + bne strcmp_unaligned + +word_aligned: + /* The strings are word-aligned. */ + /* Is the first string double-word aligned? */ + tst r0, #4 + beq doubleword_aligned_r0 + + /* If r0 is not double-word aligned yet, align it by loading + and comparing the next word from each string. */ + ldr r2, [r0], #4 + ldr r4, [r1], #4 + magic_compare_and_branch w1=r2 w2=r4 label=return_24 + +doubleword_aligned_r0: + /* Get here when r0 is double-word aligned. */ + /* Is r1 doubleword_aligned? */ + tst r1, #4 + beq doubleword_aligned + + /* Get here when the strings to compare are word-aligned, + r0 is double-word aligned, but r1 is not double-word aligned. */ + + /* Initialize the queue. */ + ldr r5, [r1], #4 + + /* Compare two words in every iteration. */ + .p2align 2 +3: + pld [r0, #16] + pld [r1, #16] + + /* Load the next double-word from each string and compare. */ + ldrd r2, r3, [r0], #8 + magic_compare_and_branch w1=r2 w2=r5 label=return_25 + ldrd r4, r5, [r1], #8 + magic_compare_and_branch w1=r3 w2=r4 label=return_34 + b 3b + + .macro miscmp_word offsetlo offsethi + /* Macro to compare misaligned strings. */ + /* r0, r1 are word-aligned, and at least one of the strings + is not double-word aligned. */ + /* Compare one word in every loop iteration. */ + /* OFFSETLO is the original bit-offset of r1 from a word-boundary, + OFFSETHI is 32 - OFFSETLO (i.e., offset from the next word). */ + + /* Initialize the shift queue. */ + ldr r5, [r1], #4 + + /* Compare one word from each string in every loop iteration. */ + .p2align 2 +7: + ldr r3, [r0], #4 + S2LOMEM r5, r5, #\offsetlo + magic_find_zero_bytes w1=r3 + cmp r7, ip, S2HIMEM #\offsetlo + and r2, r3, r6, S2LOMEM #\offsetlo + it eq + cmpeq r2, r5 + bne return_25 + ldr r5, [r1], #4 + cmp ip, #0 + eor r3, r2, r3 + S2HIMEM r2, r5, #\offsethi + it eq + cmpeq r3, r2 + bne return_32 + b 7b + .endm /* miscmp_word */ + +return_32: + setup_return w1=r3, w2=r2 + b do_return +return_34: + setup_return w1=r3, w2=r4 + b do_return +return_25: + setup_return w1=r2, w2=r5 + b do_return +return_35: + setup_return w1=r3, w2=r5 + b do_return +return_24: + setup_return w1=r2, w2=r4 + +do_return: + +#ifdef __ARMEB__ + mov r0, ip +#else /* not __ARMEB__ */ + rev r0, ip +#endif /* not __ARMEB__ */ + + /* Restore temporaries early, before computing the return value. */ + ldrd r6, r7, [sp] + ldrd r4, r5, [sp, #8] + adds sp, sp, #16 + + /* There is a zero or a different byte between r1 and r2. */ + /* r0 contains a mask of all-zero bytes in r1. */ + /* Using r0 and not ip here because cbz requires low register. */ + m_cbz reg=r0, label=compute_return_value + clz r0, r0 + /* r0 contains the number of bits on the left of the first all-zero byte in r1. */ + rsb r0, r0, #24 + /* Here, r0 contains the number of bits on the right of the first all-zero byte in r1. */ + lsr r1, r1, r0 + lsr r2, r2, r0 + +compute_return_value: + movs r0, #1 + cmp r1, r2 + /* The return value is computed as follows. + If r1>r2 then (C==1 and Z==0) and LS doesn't hold and r0 is #1 at return. + If r1 +#include + + .text + +#ifdef __ARMEB__ +#define SHFT2LSB lsl +#define SHFT2LSBEQ lsleq +#define SHFT2MSB lsr +#define SHFT2MSBEQ lsreq +#define MSB 0x000000ff +#define LSB 0xff000000 +#else +#define SHFT2LSB lsr +#define SHFT2LSBEQ lsreq +#define SHFT2MSB lsl +#define SHFT2MSBEQ lsleq +#define MSB 0xff000000 +#define LSB 0x000000ff +#endif + +#define magic1(REG) REG +#define magic2(REG) REG, lsl #7 + +ENTRY(strcmp) + PLD(r0, #0) + PLD(r1, #0) + eor r2, r0, r1 + tst r2, #3 + + /* Strings not at same byte offset from a word boundary. */ + bne .Lstrcmp_unaligned + ands r2, r0, #3 + bic r0, r0, #3 + bic r1, r1, #3 + ldr ip, [r0], #4 + it eq + ldreq r3, [r1], #4 + beq 1f + + /* Although s1 and s2 have identical initial alignment, they are + * not currently word aligned. Rather than comparing bytes, + * make sure that any bytes fetched from before the addressed + * bytes are forced to 0xff. Then they will always compare + * equal. + */ + eor r2, r2, #3 + lsl r2, r2, #3 + mvn r3, #MSB + SHFT2LSB r2, r3, r2 + ldr r3, [r1], #4 + orr ip, ip, r2 + orr r3, r3, r2 +1: + /* Load the 'magic' constant 0x01010101. */ + str r4, [sp, #-4]! + mov r4, #1 + orr r4, r4, r4, lsl #8 + orr r4, r4, r4, lsl #16 + .p2align 2 +4: + PLD(r0, #8) + PLD(r1, #8) + sub r2, ip, magic1(r4) + cmp ip, r3 + itttt eq + + /* check for any zero bytes in first word */ + biceq r2, r2, ip + tsteq r2, magic2(r4) + ldreq ip, [r0], #4 + ldreq r3, [r1], #4 + beq 4b +2: + /* There's a zero or a different byte in the word */ + SHFT2MSB r0, ip, #24 + SHFT2LSB ip, ip, #8 + cmp r0, #1 + it cs + cmpcs r0, r3, SHFT2MSB #24 + it eq + SHFT2LSBEQ r3, r3, #8 + beq 2b + /* On a big-endian machine, r0 contains the desired byte in bits + * 0-7; on a little-endian machine they are in bits 24-31. In + * both cases the other bits in r0 are all zero. For r3 the + * interesting byte is at the other end of the word, but the + * other bits are not necessarily zero. We need a signed result + * representing the differnece in the unsigned bytes, so for the + * little-endian case we can't just shift the interesting bits up. + */ +#ifdef __ARMEB__ + sub r0, r0, r3, lsr #24 +#else + and r3, r3, #255 + /* No RSB instruction in Thumb2 */ +#ifdef __thumb2__ + lsr r0, r0, #24 + sub r0, r0, r3 +#else + rsb r0, r3, r0, lsr #24 +#endif +#endif + ldr r4, [sp], #4 + bx lr + +.Lstrcmp_unaligned: + wp1 .req r0 + wp2 .req r1 + b1 .req r2 + w1 .req r4 + w2 .req r5 + t1 .req ip + @ r3 is scratch + + /* First of all, compare bytes until wp1(sp1) is word-aligned. */ +1: + tst wp1, #3 + beq 2f + ldrb r2, [wp1], #1 + ldrb r3, [wp2], #1 + cmp r2, #1 + it cs + cmpcs r2, r3 + beq 1b + sub r0, r2, r3 + bx lr + +2: + str r5, [sp, #-4]! + str r4, [sp, #-4]! + mov b1, #1 + orr b1, b1, b1, lsl #8 + orr b1, b1, b1, lsl #16 + + and t1, wp2, #3 + bic wp2, wp2, #3 + ldr w1, [wp1], #4 + ldr w2, [wp2], #4 + cmp t1, #2 + beq 2f + bhi 3f + + /* Critical inner Loop: Block with 3 bytes initial overlap */ + .p2align 2 +1: + bic t1, w1, #MSB + cmp t1, w2, SHFT2LSB #8 + sub r3, w1, b1 + bic r3, r3, w1 + bne 4f + ands r3, r3, b1, lsl #7 + it eq + ldreq w2, [wp2], #4 + bne 5f + eor t1, t1, w1 + cmp t1, w2, SHFT2MSB #24 + bne 6f + ldr w1, [wp1], #4 + b 1b +4: + SHFT2LSB w2, w2, #8 + b 8f + +5: +#ifdef __ARMEB__ + /* The syndrome value may contain false ones if the string ends + * with the bytes 0x01 0x00 + */ + tst w1, #0xff000000 + itt ne + tstne w1, #0x00ff0000 + tstne w1, #0x0000ff00 + beq 7f +#else + bics r3, r3, #0xff000000 + bne 7f +#endif + ldrb w2, [wp2] + SHFT2LSB t1, w1, #24 +#ifdef __ARMEB__ + lsl w2, w2, #24 +#endif + b 8f + +6: + SHFT2LSB t1, w1, #24 + and w2, w2, #LSB + b 8f + + /* Critical inner Loop: Block with 2 bytes initial overlap */ + .p2align 2 +2: + SHFT2MSB t1, w1, #16 + sub r3, w1, b1 + SHFT2LSB t1, t1, #16 + bic r3, r3, w1 + cmp t1, w2, SHFT2LSB #16 + bne 4f + ands r3, r3, b1, lsl #7 + it eq + ldreq w2, [wp2], #4 + bne 5f + eor t1, t1, w1 + cmp t1, w2, SHFT2MSB #16 + bne 6f + ldr w1, [wp1], #4 + b 2b + +5: +#ifdef __ARMEB__ + /* The syndrome value may contain false ones if the string ends + * with the bytes 0x01 0x00 + */ + tst w1, #0xff000000 + it ne + tstne w1, #0x00ff0000 + beq 7f +#else + lsls r3, r3, #16 + bne 7f +#endif + ldrh w2, [wp2] + SHFT2LSB t1, w1, #16 +#ifdef __ARMEB__ + lsl w2, w2, #16 +#endif + b 8f + +6: + SHFT2MSB w2, w2, #16 + SHFT2LSB t1, w1, #16 +4: + SHFT2LSB w2, w2, #16 + b 8f + + /* Critical inner Loop: Block with 1 byte initial overlap */ + .p2align 2 +3: + and t1, w1, #LSB + cmp t1, w2, SHFT2LSB #24 + sub r3, w1, b1 + bic r3, r3, w1 + bne 4f + ands r3, r3, b1, lsl #7 + it eq + ldreq w2, [wp2], #4 + bne 5f + eor t1, t1, w1 + cmp t1, w2, SHFT2MSB #8 + bne 6f + ldr w1, [wp1], #4 + b 3b +4: + SHFT2LSB w2, w2, #24 + b 8f +5: + /* The syndrome value may contain false ones if the string ends + * with the bytes 0x01 0x00 + */ + tst w1, #LSB + beq 7f + ldr w2, [wp2], #4 +6: + SHFT2LSB t1, w1, #8 + bic w2, w2, #MSB + b 8f +7: + mov r0, #0 + ldr r4, [sp], #4 + ldr r5, [sp], #4 + bx lr + +8: + and r2, t1, #LSB + and r0, w2, #LSB + cmp r0, #1 + it cs + cmpcs r0, r2 + itt eq + SHFT2LSBEQ t1, t1, #8 + SHFT2LSBEQ w2, w2, #8 + beq 8b + sub r0, r2, r0 + ldr r4, [sp], #4 + ldr r5, [sp], #4 + bx lr +END(strcmp) diff --git a/libc/arch-arm/generic/generic.mk b/libc/arch-arm/generic/generic.mk index 46bd0671d..358b1e6f6 100644 --- a/libc/arch-arm/generic/generic.mk +++ b/libc/arch-arm/generic/generic.mk @@ -1,2 +1,3 @@ $(call libc-add-cpu-variant-src,MEMCPY,arch-arm/generic/bionic/memcpy.S) $(call libc-add-cpu-variant-src,MEMSET,arch-arm/generic/bionic/memset.S) +$(call libc-add-cpu-variant-src,STRCMP,arch-arm/generic/bionic/strcmp.S) diff --git a/libc/arch-arm/krait/bionic/strcmp.S b/libc/arch-arm/krait/bionic/strcmp.S new file mode 100644 index 000000000..d614b9db0 --- /dev/null +++ b/libc/arch-arm/krait/bionic/strcmp.S @@ -0,0 +1,477 @@ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#ifdef __ARMEB__ +#define S2LOMEM lsl +#define S2LOMEMEQ lsleq +#define S2HIMEM lsr +#define MSB 0x000000ff +#define LSB 0xff000000 +#define BYTE0_OFFSET 24 +#define BYTE1_OFFSET 16 +#define BYTE2_OFFSET 8 +#define BYTE3_OFFSET 0 +#else /* not __ARMEB__ */ +#define S2LOMEM lsr +#define S2LOMEMEQ lsreq +#define S2HIMEM lsl +#define BYTE0_OFFSET 0 +#define BYTE1_OFFSET 8 +#define BYTE2_OFFSET 16 +#define BYTE3_OFFSET 24 +#define MSB 0xff000000 +#define LSB 0x000000ff +#endif /* not __ARMEB__ */ + +.syntax unified + +#if defined (__thumb__) + .thumb + .thumb_func +#endif + +ENTRY(strcmp) + /* Use LDRD whenever possible. */ + +/* The main thing to look out for when comparing large blocks is that + the loads do not cross a page boundary when loading past the index + of the byte with the first difference or the first string-terminator. + + For example, if the strings are identical and the string-terminator + is at index k, byte by byte comparison will not load beyond address + s1+k and s2+k; word by word comparison may load up to 3 bytes beyond + k; double word - up to 7 bytes. If the load of these bytes crosses + a page boundary, it might cause a memory fault (if the page is not mapped) + that would not have happened in byte by byte comparison. + + If an address is (double) word aligned, then a load of a (double) word + from that address will not cross a page boundary. + Therefore, the algorithm below considers word and double-word alignment + of strings separately. */ + +/* High-level description of the algorithm. + + * The fast path: if both strings are double-word aligned, + use LDRD to load two words from each string in every loop iteration. + * If the strings have the same offset from a word boundary, + use LDRB to load and compare byte by byte until + the first string is aligned to a word boundary (at most 3 bytes). + This is optimized for quick return on short unaligned strings. + * If the strings have the same offset from a double-word boundary, + use LDRD to load two words from each string in every loop iteration, as in the fast path. + * If the strings do not have the same offset from a double-word boundary, + load a word from the second string before the loop to initialize the queue. + Use LDRD to load two words from every string in every loop iteration. + Inside the loop, load the second word from the second string only after comparing + the first word, using the queued value, to guarantee safety across page boundaries. + * If the strings do not have the same offset from a word boundary, + use LDR and a shift queue. Order of loads and comparisons matters, + similarly to the previous case. + + * Use UADD8 and SEL to compare words, and use REV and CLZ to compute the return value. + * The only difference between ARM and Thumb modes is the use of CBZ instruction. + * The only difference between big and little endian is the use of REV in little endian + to compute the return value, instead of MOV. +*/ + + .macro m_cbz reg label +#ifdef __thumb2__ + cbz \reg, \label +#else /* not defined __thumb2__ */ + cmp \reg, #0 + beq \label +#endif /* not defined __thumb2__ */ + .endm /* m_cbz */ + + .macro m_cbnz reg label +#ifdef __thumb2__ + cbnz \reg, \label +#else /* not defined __thumb2__ */ + cmp \reg, #0 + bne \label +#endif /* not defined __thumb2__ */ + .endm /* m_cbnz */ + + .macro init + /* Macro to save temporary registers and prepare magic values. */ + subs sp, sp, #16 + strd r4, r5, [sp, #8] + strd r6, r7, [sp] + mvn r6, #0 /* all F */ + mov r7, #0 /* all 0 */ + .endm /* init */ + + .macro magic_compare_and_branch w1 w2 label + /* Macro to compare registers w1 and w2 and conditionally branch to label. */ + cmp \w1, \w2 /* Are w1 and w2 the same? */ + magic_find_zero_bytes \w1 + it eq + cmpeq ip, #0 /* Is there a zero byte in w1? */ + bne \label + .endm /* magic_compare_and_branch */ + + .macro magic_find_zero_bytes w1 + /* Macro to find all-zero bytes in w1, result is in ip. */ +#if (defined (__ARM_FEATURE_DSP)) + uadd8 ip, \w1, r6 + sel ip, r7, r6 +#else /* not defined (__ARM_FEATURE_DSP) */ + /* __ARM_FEATURE_DSP is not defined for some Cortex-M processors. + Coincidently, these processors only have Thumb-2 mode, where we can use the + the (large) magic constant available directly as an immediate in instructions. + Note that we cannot use the magic constant in ARM mode, where we need + to create the constant in a register. */ + sub ip, \w1, #0x01010101 + bic ip, ip, \w1 + and ip, ip, #0x80808080 +#endif /* not defined (__ARM_FEATURE_DSP) */ + .endm /* magic_find_zero_bytes */ + + .macro setup_return w1 w2 +#ifdef __ARMEB__ + mov r1, \w1 + mov r2, \w2 +#else /* not __ARMEB__ */ + rev r1, \w1 + rev r2, \w2 +#endif /* not __ARMEB__ */ + .endm /* setup_return */ + + pld [r0, #0] + pld [r1, #0] + + /* Are both strings double-word aligned? */ + orr ip, r0, r1 + tst ip, #7 + bne do_align + + /* Fast path. */ + init + +doubleword_aligned: + + /* Get here when the strings to compare are double-word aligned. */ + /* Compare two words in every iteration. */ + .p2align 2 +2: + pld [r0, #16] + pld [r1, #16] + + /* Load the next double-word from each string. */ + ldrd r2, r3, [r0], #8 + ldrd r4, r5, [r1], #8 + + magic_compare_and_branch w1=r2, w2=r4, label=return_24 + magic_compare_and_branch w1=r3, w2=r5, label=return_35 + b 2b + +do_align: + /* Is the first string word-aligned? */ + ands ip, r0, #3 + beq word_aligned_r0 + + /* Fast compare byte by byte until the first string is word-aligned. */ + /* The offset of r0 from a word boundary is in ip. Thus, the number of bytes + to read until the next word boundary is 4-ip. */ + bic r0, r0, #3 + ldr r2, [r0], #4 + lsls ip, ip, #31 + beq byte2 + bcs byte3 + +byte1: + ldrb ip, [r1], #1 + uxtb r3, r2, ror #BYTE1_OFFSET + subs ip, r3, ip + bne fast_return + m_cbz reg=r3, label=fast_return + +byte2: + ldrb ip, [r1], #1 + uxtb r3, r2, ror #BYTE2_OFFSET + subs ip, r3, ip + bne fast_return + m_cbz reg=r3, label=fast_return + +byte3: + ldrb ip, [r1], #1 + uxtb r3, r2, ror #BYTE3_OFFSET + subs ip, r3, ip + bne fast_return + m_cbnz reg=r3, label=word_aligned_r0 + +fast_return: + mov r0, ip + bx lr + +word_aligned_r0: + init + /* The first string is word-aligned. */ + /* Is the second string word-aligned? */ + ands ip, r1, #3 + bne strcmp_unaligned + +word_aligned: + /* The strings are word-aligned. */ + /* Is the first string double-word aligned? */ + tst r0, #4 + beq doubleword_aligned_r0 + + /* If r0 is not double-word aligned yet, align it by loading + and comparing the next word from each string. */ + ldr r2, [r0], #4 + ldr r4, [r1], #4 + magic_compare_and_branch w1=r2 w2=r4 label=return_24 + +doubleword_aligned_r0: + /* Get here when r0 is double-word aligned. */ + /* Is r1 doubleword_aligned? */ + tst r1, #4 + beq doubleword_aligned + + /* Get here when the strings to compare are word-aligned, + r0 is double-word aligned, but r1 is not double-word aligned. */ + + /* Initialize the queue. */ + ldr r5, [r1], #4 + + /* Compare two words in every iteration. */ + .p2align 2 +3: + pld [r0, #16] + pld [r1, #16] + + /* Load the next double-word from each string and compare. */ + ldrd r2, r3, [r0], #8 + magic_compare_and_branch w1=r2 w2=r5 label=return_25 + ldrd r4, r5, [r1], #8 + magic_compare_and_branch w1=r3 w2=r4 label=return_34 + b 3b + + .macro miscmp_word offsetlo offsethi + /* Macro to compare misaligned strings. */ + /* r0, r1 are word-aligned, and at least one of the strings + is not double-word aligned. */ + /* Compare one word in every loop iteration. */ + /* OFFSETLO is the original bit-offset of r1 from a word-boundary, + OFFSETHI is 32 - OFFSETLO (i.e., offset from the next word). */ + + /* Initialize the shift queue. */ + ldr r5, [r1], #4 + + /* Compare one word from each string in every loop iteration. */ + .p2align 2 +7: + ldr r3, [r0], #4 + S2LOMEM r5, r5, #\offsetlo + magic_find_zero_bytes w1=r3 + cmp r7, ip, S2HIMEM #\offsetlo + and r2, r3, r6, S2LOMEM #\offsetlo + it eq + cmpeq r2, r5 + bne return_25 + ldr r5, [r1], #4 + cmp ip, #0 + eor r3, r2, r3 + S2HIMEM r2, r5, #\offsethi + it eq + cmpeq r3, r2 + bne return_32 + b 7b + .endm /* miscmp_word */ + +strcmp_unaligned: + /* r0 is word-aligned, r1 is at offset ip from a word. */ + /* Align r1 to the (previous) word-boundary. */ + bic r1, r1, #3 + + /* Unaligned comparison word by word using LDRs. */ + cmp ip, #2 + beq miscmp_word_16 /* If ip == 2. */ + bge miscmp_word_24 /* If ip == 3. */ + miscmp_word offsetlo=8 offsethi=24 /* If ip == 1. */ +miscmp_word_24: miscmp_word offsetlo=24 offsethi=8 + + +return_32: + setup_return w1=r3, w2=r2 + b do_return +return_34: + setup_return w1=r3, w2=r4 + b do_return +return_25: + setup_return w1=r2, w2=r5 + b do_return +return_35: + setup_return w1=r3, w2=r5 + b do_return +return_24: + setup_return w1=r2, w2=r4 + +do_return: + +#ifdef __ARMEB__ + mov r0, ip +#else /* not __ARMEB__ */ + rev r0, ip +#endif /* not __ARMEB__ */ + + /* Restore temporaries early, before computing the return value. */ + ldrd r6, r7, [sp] + ldrd r4, r5, [sp, #8] + adds sp, sp, #16 + + /* There is a zero or a different byte between r1 and r2. */ + /* r0 contains a mask of all-zero bytes in r1. */ + /* Using r0 and not ip here because cbz requires low register. */ + m_cbz reg=r0, label=compute_return_value + clz r0, r0 + /* r0 contains the number of bits on the left of the first all-zero byte in r1. */ + rsb r0, r0, #24 + /* Here, r0 contains the number of bits on the right of the first all-zero byte in r1. */ + lsr r1, r1, r0 + lsr r2, r2, r0 + +compute_return_value: + movs r0, #1 + cmp r1, r2 + /* The return value is computed as follows. + If r1>r2 then (C==1 and Z==0) and LS doesn't hold and r0 is #1 at return. + If r1 Date: Fri, 15 Mar 2013 16:01:17 -0700 Subject: [PATCH 05/77] Update to latest cortexa15 memcpy code. This uses the new code original submitted as memcpy.a15.S as the base. However, the old code handled unaligned src/dst better so that was spliced in. I optimized the original unaligned code by removing a few unnecessary instructions. I optimized the a15 code by rewriting the pre and post code. I also modified the main loop to add a pld so that larger copies would not stall waiting for memory. Test cases for the new memcpy: - Copy all sized values from 0 to 1024 bytes, using whatever alignment is returned by malloc. For each alignment case described below, the test copied from 0 to 128 bytes. - Src and dst pointers are both aligned to the same value, starting at one going through every power of two up to and including 128. - Src aligned to double word boundary, dst aligned to word boundary. - Src aligned to word boundary, dst aligned to double word boundary. - Src aligned to 16 bit boundary, dst aligned to word boundary. - Src aligned to word boundary, dst aligned to 16 byte boundary. - Src aligned to word boundary, dst aligned to 1 byte from a word boundary. - Src aligned to word boundary, dst aligned to 2 bytes from a word boundary. - Src aligned to word boundary, dst aligned to 3 bytes from a word boundary. - Src aligned to 1 byte from a word boundary, dst aligned to a word boundary. - Src aligned to 2 bytes from a word boundary, dst aligned to a word boundary. - Src aligned to 3 bytes from a word boundary, dst aligned to a word boundary. Cases to verify the unaligned source code properly aligns to a 16 bit boundary. - Src aligned to 1 byte from a 128 bit boundary, dst aligned to 4 + 128 bit boundary. - Src aligned to 1 byte from a 128 bit boundary, dst aligned to 8 + 128 bit boundary. - Src aligned to 1 byte from a 128 bit boundary, dst aligned to 12 + 128 bit boundary. - Src aligned to 1 byte from a 128 bit boundary, dst aligned to 16 + 128 bit boundary. In all cases, a two byte fencepost was placed at the end of the destination to verify that only the requested number of bytes were copied. Bug: 8005082 Change-Id: I700b2fab81941959d301ab1934c18fbd8ee3eee4 --- libc/arch-arm/cortex-a15/bionic/memcpy.S | 311 ++++++++++++++++++----- 1 file changed, 245 insertions(+), 66 deletions(-) diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy.S b/libc/arch-arm/cortex-a15/bionic/memcpy.S index 16187b562..9985e7f42 100644 --- a/libc/arch-arm/cortex-a15/bionic/memcpy.S +++ b/libc/arch-arm/cortex-a15/bionic/memcpy.S @@ -24,81 +24,110 @@ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* Assumes neon instructions and a cache line size of 64 bytes. */ + /* Prototype: void *memcpy (void *dst, const void *src, size_t count). */ + + // This version is tuned for the Cortex-A15 processor. #include #include -/* - * This code assumes it is running on a processor that supports all arm v7 - * instructions, that supports neon instructions, and that has a 64 byte - * cache line. - */ - .text + .syntax unified .fpu neon -#define CACHE_LINE_SIZE 64 +#define CACHE_LINE_SIZE 64 ENTRY(memcpy) - .save {r0, lr} - /* start preloading as early as possible */ - pld [r1, #(CACHE_LINE_SIZE*0)] - stmfd sp!, {r0, lr} - pld [r1, #(CACHE_LINE_SIZE*1)] + // Assumes that n >= 0, and dst, src are valid pointers. + // For any sizes less than 832 use the neon code that doesn't + // care about the src alignment. This avoids any checks + // for src alignment, and offers the best improvement since + // smaller sized copies are dominated by the overhead of + // the pre and post main loop. + // For larger copies, if src and dst cannot both be aligned to + // word boundaries, use the neon code. + // For all other copies, align dst to a double word boundary + // and copy using LDRD/STRD instructions. - /* do we have at least 16-bytes to copy (needed for alignment below) */ - cmp r2, #16 - blo 5f + // Save registers (r0 holds the return value): + // optimized push {r0, lr}. + .save {r0, lr} + pld [r1, #(CACHE_LINE_SIZE*16)] + push {r0, lr} - /* align destination to cache-line for the write-buffer */ + cmp r2, #16 + blo copy_less_than_16_unknown_align + + cmp r2, #832 + bge check_alignment + +copy_unknown_alignment: + // Unknown alignment of src and dst. + // Assumes that the first few bytes have already been prefetched. + + // Align destination to 128 bits. The mainloop store instructions + // require this alignment or they will throw an exception. rsb r3, r0, #0 ands r3, r3, #0xF - beq 0f + beq 2f - /* copy up to 15-bytes (count in r3) */ + // Copy up to 15 bytes (count in r3). sub r2, r2, r3 movs ip, r3, lsl #31 - ldrmib lr, [r1], #1 - strmib lr, [r0], #1 - ldrcsb ip, [r1], #1 - ldrcsb lr, [r1], #1 - strcsb ip, [r0], #1 - strcsb lr, [r0], #1 + + itt mi + ldrbmi lr, [r1], #1 + strbmi lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1], #1 + strbcs ip, [r0], #1 + strbcs lr, [r0], #1 + movs ip, r3, lsl #29 bge 1f - // copies 4 bytes, destination 32-bits aligned + // Copies 4 bytes, dst 32 bits aligned before, at least 64 bits after. vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]! 1: bcc 2f - // copies 8 bytes, destination 64-bits aligned + // Copies 8 bytes, dst 64 bits aligned before, at least 128 bits after. vld1.8 {d0}, [r1]! vst1.8 {d0}, [r0, :64]! -2: -0: /* preload immediately the next cache line, which we may need */ - pld [r1, #(CACHE_LINE_SIZE*0)] - pld [r1, #(CACHE_LINE_SIZE*1)] - - /* make sure we have at least 64 bytes to copy */ +2: // Make sure we have at least 64 bytes to copy. subs r2, r2, #64 blo 2f - /* Preload all the cache lines we need. - * NOTE: The number of pld below depends on CACHE_LINE_SIZE, - * ideally we would increase the distance in the main loop to - * avoid the goofy code below. In practice this doesn't seem to make - * a big difference. - * NOTE: The value CACHE_LINE_SIZE * 4 was chosen through - * experimentation. - */ - pld [r1, #(CACHE_LINE_SIZE*2)] - pld [r1, #(CACHE_LINE_SIZE*3)] - pld [r1, #(CACHE_LINE_SIZE*4)] - -1: /* The main loop copies 64 bytes at a time */ +1: // The main loop copies 64 bytes at a time. vld1.8 {d0 - d3}, [r1]! vld1.8 {d4 - d7}, [r1]! pld [r1, #(CACHE_LINE_SIZE*4)] @@ -107,25 +136,24 @@ ENTRY(memcpy) vst1.8 {d4 - d7}, [r0, :128]! bhs 1b -2: /* fix-up the remaining count and make sure we have >= 32 bytes left */ - add r2, r2, #64 - subs r2, r2, #32 - blo 4f +2: // Fix-up the remaining count and make sure we have >= 32 bytes left. + adds r2, r2, #32 + blo 3f -3: /* 32 bytes at a time. These cache lines were already preloaded */ + // 32 bytes. These cache lines were already preloaded. vld1.8 {d0 - d3}, [r1]! - subs r2, r2, #32 + sub r2, r2, #32 vst1.8 {d0 - d3}, [r0, :128]! - bhs 3b -4: /* less than 32 left */ +3: // Less than 32 left. add r2, r2, #32 tst r2, #0x10 - beq 5f - // copies 16 bytes, 128-bits aligned + beq copy_less_than_16_unknown_align + // Copies 16 bytes, destination 128 bits aligned. vld1.8 {d0, d1}, [r1]! vst1.8 {d0, d1}, [r0, :128]! -5: /* copy up to 15-bytes (count in r2) */ +copy_less_than_16_unknown_align: + // Copy up to 15 bytes (count in r2). movs ip, r2, lsl #29 bcc 1f vld1.8 {d0}, [r1]! @@ -133,14 +161,165 @@ ENTRY(memcpy) 1: bge 2f vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]! -2: movs ip, r2, lsl #31 - ldrmib r3, [r1], #1 - ldrcsb ip, [r1], #1 - ldrcsb lr, [r1], #1 - strmib r3, [r0], #1 - strcsb ip, [r0], #1 - strcsb lr, [r0], #1 - ldmfd sp!, {r0, lr} - bx lr +2: // Copy 0 to 4 bytes. + lsls r2, r2, #31 + itt ne + ldrbne lr, [r1], #1 + strbne lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1] + strbcs ip, [r0], #1 + strbcs lr, [r0] + + pop {r0, pc} + +check_alignment: + // If src and dst cannot both be aligned to a word boundary, + // use the unaligned copy version. + eor r3, r0, r1 + ands r3, r3, #0x3 + bne copy_unknown_alignment + + // To try and improve performance, stack layout changed, + // i.e., not keeping the stack looking like users expect + // (highest numbered register at highest address). + // TODO: Add debug frame directives. + // We don't need exception unwind directives, because the code below + // does not throw any exceptions and does not call any other functions. + // Generally, newlib functions like this lack debug information for + // assembler source. + .save {r4, r5} + strd r4, r5, [sp, #-8]! + .save {r6, r7} + strd r6, r7, [sp, #-8]! + .save {r8, r9} + strd r8, r9, [sp, #-8]! + + // Optimized for already aligned dst code. + ands ip, r0, #3 + bne dst_not_word_aligned + +word_aligned: + // Align the destination buffer to 8 bytes, to make sure double + // loads and stores don't cross a cache line boundary, + // as they are then more expensive even if the data is in the cache + // (require two load/store issue cycles instead of one). + // If only one of the buffers is not 8 bytes aligned, + // then it's more important to align dst than src, + // because there is more penalty for stores + // than loads that cross a cacheline boundary. + // This check and realignment are only done if there is >= 832 + // bytes to copy. + + // Dst is word aligned, but check if it is already double word aligned. + ands r3, r0, #4 + beq 1f + ldr r3, [r1], #4 + str r3, [r0], #4 + sub r2, #4 + +1: // Can only get here if > 64 bytes to copy, so don't do check r2. + sub r2, #64 + +2: // Every loop iteration copies 64 bytes. + .irp offset, #0, #8, #16, #24, #32 + ldrd r4, r5, [r1, \offset] + strd r4, r5, [r0, \offset] + .endr + + ldrd r4, r5, [r1, #40] + ldrd r6, r7, [r1, #48] + ldrd r8, r9, [r1, #56] + + // Keep the pld as far from the next load as possible. + // The amount to prefetch was determined experimentally using + // large sizes, and verifying the prefetch size does not affect + // the smaller copies too much. + // WARNING: If the ldrd and strd instructions get too far away + // from each other, performance suffers. Three loads + // in a row is the best tradeoff. + pld [r1, #(CACHE_LINE_SIZE*16)] + strd r4, r5, [r0, #40] + strd r6, r7, [r0, #48] + strd r8, r9, [r0, #56] + + add r0, r0, #64 + add r1, r1, #64 + subs r2, r2, #64 + bge 2b + + // Fix-up the remaining count and make sure we have >= 32 bytes left. + adds r2, r2, #32 + blo 4f + + // Copy 32 bytes. These cache lines were already preloaded. + .irp offset, #0, #8, #16, #24 + ldrd r4, r5, [r1, \offset] + strd r4, r5, [r0, \offset] + .endr + add r1, r1, #32 + add r0, r0, #32 + sub r2, r2, #32 +4: // Less than 32 left. + add r2, r2, #32 + tst r2, #0x10 + beq 5f + // Copy 16 bytes. + .irp offset, #0, #8 + ldrd r4, r5, [r1, \offset] + strd r4, r5, [r0, \offset] + .endr + add r1, r1, #16 + add r0, r0, #16 + +5: // Copy up to 15 bytes (count in r2). + movs ip, r2, lsl #29 + bcc 1f + // Copy 8 bytes. + ldrd r4, r5, [r1], #8 + strd r4, r5, [r0], #8 +1: bge 2f + // Copy 4 bytes. + ldr r4, [r1], #4 + str r4, [r0], #4 +2: // Copy 0 to 4 bytes. + lsls r2, r2, #31 + itt ne + ldrbne lr, [r1], #1 + strbne lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1] + strbcs ip, [r0], #1 + strbcs lr, [r0] + + // Restore registers: optimized pop {r0, pc} + ldrd r8, r9, [sp], #8 + ldrd r6, r7, [sp], #8 + ldrd r4, r5, [sp], #8 + pop {r0, pc} + +dst_not_word_aligned: + // Align dst to word. + rsb ip, ip, #4 + cmp ip, #2 + + itt gt + ldrbgt lr, [r1], #1 + strbgt lr, [r0], #1 + + itt ge + ldrbge lr, [r1], #1 + strbge lr, [r0], #1 + + ldrb lr, [r1], #1 + strb lr, [r0], #1 + + sub r2, r2, ip + + // If src is not word aligned, jump to the unaligned code. + ands ip, r1, #0x3 + beq word_aligned END(memcpy) From 6ffaa931c362602a2b606a610c92326a425a876e Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Tue, 2 Apr 2013 09:19:00 -0700 Subject: [PATCH 06/77] Add missing branch in memcpy.S dst aligned case. Change-Id: I0651e46dc5d9b49a997c0c6d099b75097eedb504 --- libc/arch-arm/cortex-a15/bionic/memcpy.S | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy.S b/libc/arch-arm/cortex-a15/bionic/memcpy.S index 9985e7f42..d29706488 100644 --- a/libc/arch-arm/cortex-a15/bionic/memcpy.S +++ b/libc/arch-arm/cortex-a15/bionic/memcpy.S @@ -319,7 +319,6 @@ dst_not_word_aligned: sub r2, r2, ip - // If src is not word aligned, jump to the unaligned code. - ands ip, r1, #0x3 - beq word_aligned + // Src is guaranteed to be at least word aligned by this point. + b word_aligned END(memcpy) From 7ffad9c120054eedebd5f56f8bed01144e93eafa Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Mon, 8 Apr 2013 18:35:30 -0700 Subject: [PATCH 07/77] Rewrite memset for cortexa15 to use strd. Change-Id: Iac3af55f7813bd2b40a41bd19403f2b4dca5224b --- libc/arch-arm/cortex-a15/bionic/memset.S | 146 ++++++++++++++++------- 1 file changed, 102 insertions(+), 44 deletions(-) diff --git a/libc/arch-arm/cortex-a15/bionic/memset.S b/libc/arch-arm/cortex-a15/bionic/memset.S index 7bb329752..2e1ad54c6 100644 --- a/libc/arch-arm/cortex-a15/bionic/memset.S +++ b/libc/arch-arm/cortex-a15/bionic/memset.S @@ -1,5 +1,5 @@ /* - * Copyright (C) 2008 The Android Open Source Project + * Copyright (C) 2013 The Android Open Source Project * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -35,11 +35,12 @@ * memset() returns its first argument. */ - .fpu neon + .fpu neon + .syntax unified ENTRY(bzero) - mov r2, r1 - mov r1, #0 + mov r2, r1 + mov r1, #0 // Fall through to memset... END(bzero) @@ -47,60 +48,117 @@ ENTRY(memset) .save {r0} stmfd sp!, {r0} - vdup.8 q0, r1 - - /* do we have at least 16-bytes to write (needed for alignment below) */ + // The new algorithm is slower for copies < 16 so use the old + // neon code in that case. cmp r2, #16 - blo 3f + blo set_less_than_16_unknown_align - /* align destination to 16 bytes for the write-buffer */ - rsb r3, r0, #0 - ands r3, r3, #0xF - beq 2f + // Use strd which requires an even and odd register so move the + // values so that: + // r0 and r1 contain the memset value + // r2 is the number of bytes to set + // r3 is the destination pointer + mov r3, r0 - /* write up to 15-bytes (count in r3) */ - sub r2, r2, r3 - movs ip, r3, lsl #31 - strmib r1, [r0], #1 - strcsb r1, [r0], #1 - strcsb r1, [r0], #1 - movs ip, r3, lsl #29 - bge 1f + // Copy the byte value in every byte of r1. + mov r1, r1, lsl #24 + orr r1, r1, r1, lsr #8 + orr r1, r1, r1, lsr #16 - // writes 4 bytes, 32-bits aligned - vst1.32 {d0[0]}, [r0, :32]! -1: bcc 2f +check_alignment: + // Align destination to a double word to avoid the strd crossing + // a cache line boundary. + ands ip, r3, #7 + bne do_double_word_align - // writes 8 bytes, 64-bits aligned - vst1.8 {d0}, [r0, :64]! -2: - /* make sure we have at least 32 bytes to write */ - subs r2, r2, #32 - blo 2f - vmov q1, q0 +double_word_aligned: + mov r0, r1 -1: /* The main loop writes 32 bytes at a time */ - subs r2, r2, #32 - vst1.8 {d0 - d3}, [r0, :128]! - bhs 1b + subs r2, #64 + blo set_less_than_64 -2: /* less than 32 left */ - add r2, r2, #32 - tst r2, #0x10 - beq 3f +1: // Main loop sets 64 bytes at a time. + .irp offset, #0, #8, #16, #24, #32, #40, #48, #56 + strd r0, r1, [r3, \offset] + .endr - // writes 16 bytes, 128-bits aligned - vst1.8 {d0, d1}, [r0, :128]! -3: /* write up to 15-bytes (count in r2) */ + add r3, #64 + subs r2, #64 + bge 1b + +set_less_than_64: + // Restore r2 to the count of bytes left to set. + add r2, #64 + lsls ip, r2, #27 + bcc set_less_than_32 + // Set 32 bytes. + .irp offset, #0, #8, #16, #24 + strd r0, r1, [r3, \offset] + .endr + add r3, #32 + +set_less_than_32: + bpl set_less_than_16 + // Set 16 bytes. + .irp offset, #0, #8 + strd r0, r1, [r3, \offset] + .endr + add r3, #16 + +set_less_than_16: + // Less than 16 bytes to set. + lsls ip, r2, #29 + bcc set_less_than_8 + + // Set 8 bytes. + strd r0, r1, [r3], #8 + +set_less_than_8: + bpl set_less_than_4 + // Set 4 bytes + str r1, [r3], #4 + +set_less_than_4: + lsls ip, r2, #31 + it ne + strbne r1, [r3], #1 + itt cs + strbcs r1, [r3], #1 + strbcs r1, [r3] + + ldmfd sp!, {r0} + bx lr + +do_double_word_align: + rsb ip, ip, #8 + sub r2, r2, ip + movs r0, ip, lsl #31 + it mi + strbmi r1, [r3], #1 + itt cs + strbcs r1, [r3], #1 + strbcs r1, [r3], #1 + + // Dst is at least word aligned by this point. + cmp ip, #4 + blo double_word_aligned + str r1, [r3], #4 + b double_word_aligned + +set_less_than_16_unknown_align: + // Set up to 15 bytes. + vdup.8 d0, r1 movs ip, r2, lsl #29 bcc 1f vst1.8 {d0}, [r0]! 1: bge 2f vst1.32 {d0[0]}, [r0]! 2: movs ip, r2, lsl #31 - strmib r1, [r0], #1 - strcsb r1, [r0], #1 - strcsb r1, [r0], #1 + it mi + strbmi r1, [r0], #1 + itt cs + strbcs r1, [r0], #1 + strbcs r1, [r0], #1 ldmfd sp!, {r0} bx lr END(memset) From 4d8fe5177eae8abe3cf5a596916e85daee78a0f4 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Fri, 19 Apr 2013 14:01:50 -0700 Subject: [PATCH 08/77] Tune the memcpy for krait. Streamline the memcpy a bit removing some unnecessary instructions. The biggest speed improvement comes from changing the size of the preload. On krait, the sweet spot for the preload in the main loop is twice the L1 cache line size. In most cases, these small tweaks yield > 1000MB/s speed ups. As the size of the memcpy approaches about 1MB, the speed improvement disappears. Change-Id: Ief79694d65324e2db41bee4707dae19b8c24be62 --- libc/arch-arm/krait/bionic/memcpy.S | 37 +++++++---------------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/libc/arch-arm/krait/bionic/memcpy.S b/libc/arch-arm/krait/bionic/memcpy.S index 0cd4d445a..4a21709fb 100644 --- a/libc/arch-arm/krait/bionic/memcpy.S +++ b/libc/arch-arm/krait/bionic/memcpy.S @@ -1,5 +1,5 @@ /* - * Copyright (C) 2008 The Android Open Source Project + * Copyright (C) 2013 The Android Open Source Project * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -45,9 +45,8 @@ ENTRY(memcpy) .save {r0, lr} /* start preloading as early as possible */ - pld [r1, #(CACHE_LINE_SIZE*0)] + pld [r1, #(CACHE_LINE_SIZE*4)] stmfd sp!, {r0, lr} - pld [r1, #(CACHE_LINE_SIZE*2)] /* do we have at least 16-bytes to copy (needed for alignment below) */ cmp r2, #16 @@ -56,7 +55,7 @@ ENTRY(memcpy) /* align destination to cache-line for the write-buffer */ rsb r3, r0, #0 ands r3, r3, #0xF - beq 0f + beq 2f /* copy up to 15-bytes (count in r3) */ sub r2, r2, r3 @@ -76,47 +75,29 @@ ENTRY(memcpy) // copies 8 bytes, destination 64-bits aligned vld1.8 {d0}, [r1]! vst1.8 {d0}, [r0, :64]! -2: -0: /* preload immediately the next cache line, which we may need */ - pld [r1, #(CACHE_LINE_SIZE*0)] - pld [r1, #(CACHE_LINE_SIZE*2)] - - /* make sure we have at least 64 bytes to copy */ +2: /* make sure we have at least 64 bytes to copy */ subs r2, r2, #64 blo 2f - /* Preload all the cache lines we need. - * NOTE: The number of pld below depends on CACHE_LINE_SIZE, - * ideally we would increase the distance in the main loop to - * avoid the goofy code below. In practice this doesn't seem to make - * a big difference. - * NOTE: The value CACHE_LINE_SIZE * 8 was chosen through - * experimentation. - */ - pld [r1, #(CACHE_LINE_SIZE*4)] - pld [r1, #(CACHE_LINE_SIZE*6)] - pld [r1, #(CACHE_LINE_SIZE*8)] - 1: /* The main loop copies 64 bytes at a time */ vld1.8 {d0 - d3}, [r1]! vld1.8 {d4 - d7}, [r1]! - pld [r1, #(CACHE_LINE_SIZE*8)] + pld [r1, #(CACHE_LINE_SIZE*2)] subs r2, r2, #64 vst1.8 {d0 - d3}, [r0, :128]! vst1.8 {d4 - d7}, [r0, :128]! bhs 1b 2: /* fix-up the remaining count and make sure we have >= 32 bytes left */ - add r2, r2, #64 - subs r2, r2, #32 + adds r2, r2, #32 blo 4f -3: /* 32 bytes at a time. These cache lines were already preloaded */ + /* Copy 32 bytes. These cache lines were already preloaded */ vld1.8 {d0 - d3}, [r1]! - subs r2, r2, #32 + sub r2, r2, #32 vst1.8 {d0 - d3}, [r0, :128]! - bhs 3b + 4: /* less than 32 left */ add r2, r2, #32 tst r2, #0x10 From b928bda83d4413b703329f607e2706602f15293f Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 9 May 2013 15:56:23 -0700 Subject: [PATCH 09/77] libc: add clock ids CLOCK_REALTIME_ALARM and CLOCK_BOOTTIME_ALARM Change-Id: I3bf934ab207e39b435e6b41a5fbe7eb318496dc0 --- libc/include/time.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libc/include/time.h b/libc/include/time.h index e280e0af1..edcc5bdee 100644 --- a/libc/include/time.h +++ b/libc/include/time.h @@ -104,6 +104,8 @@ extern int clock_gettime(int, struct timespec *); #define CLOCK_REALTIME_HR 4 #define CLOCK_MONOTONIC_HR 5 #define CLOCK_BOOTTIME 7 +#define CLOCK_REALTIME_ALARM 8 +#define CLOCK_BOOTTIME_ALARM 9 extern int timer_create(int, struct sigevent*, timer_t*); extern int timer_delete(timer_t); From 60e5144ca312b210b54ac8e6966108da0c97ff80 Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Fri, 10 May 2013 18:23:40 -0700 Subject: [PATCH 10/77] libc: remove obsolete CLOCK_REALTIME_HR and CLOCK_MONOTONIC_HR Add CLOCK_MONOTONIC_RAW, CLOCK_REALTIME_COARSE, and CLOCK_MONOTONIC_COARSE as supported by recent linux kernels. Bug: 8895727 Change-Id: I3d415a2edbcf2928dd855e876337bf2239ac134a --- libc/include/time.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libc/include/time.h b/libc/include/time.h index edcc5bdee..8995585a0 100644 --- a/libc/include/time.h +++ b/libc/include/time.h @@ -101,8 +101,9 @@ extern int clock_gettime(int, struct timespec *); #define CLOCK_MONOTONIC 1 #define CLOCK_PROCESS_CPUTIME_ID 2 #define CLOCK_THREAD_CPUTIME_ID 3 -#define CLOCK_REALTIME_HR 4 -#define CLOCK_MONOTONIC_HR 5 +#define CLOCK_MONOTONIC_RAW 4 +#define CLOCK_REALTIME_COARSE 5 +#define CLOCK_MONOTONIC_COARSE 6 #define CLOCK_BOOTTIME 7 #define CLOCK_REALTIME_ALARM 8 #define CLOCK_BOOTTIME_ALARM 9 From 04c0ac14a49e0969333008a9522b64046d58fbdc Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Wed, 8 May 2013 20:21:00 -0700 Subject: [PATCH 11/77] libc: add timerfd calls Change-Id: Id63b907266d5b87c7422a51d393a1430551ca33d --- libc/SYSCALLS.TXT | 3 ++ libc/arch-arm/syscalls.mk | 3 ++ libc/arch-arm/syscalls/timerfd_create.S | 15 ++++++++ libc/arch-arm/syscalls/timerfd_gettime.S | 15 ++++++++ libc/arch-arm/syscalls/timerfd_settime.S | 15 ++++++++ libc/arch-mips/syscalls.mk | 3 ++ libc/arch-mips/syscalls/timerfd_create.S | 22 +++++++++++ libc/arch-mips/syscalls/timerfd_gettime.S | 22 +++++++++++ libc/arch-mips/syscalls/timerfd_settime.S | 22 +++++++++++ libc/arch-x86/syscalls.mk | 3 ++ libc/arch-x86/syscalls/timerfd_create.S | 24 ++++++++++++ libc/arch-x86/syscalls/timerfd_gettime.S | 24 ++++++++++++ libc/arch-x86/syscalls/timerfd_settime.S | 30 +++++++++++++++ libc/include/sys/timerfd.h | 45 +++++++++++++++++++++++ libc/kernel/common/linux/timerfd.h | 31 ++++++++++++++++ 15 files changed, 277 insertions(+) create mode 100644 libc/arch-arm/syscalls/timerfd_create.S create mode 100644 libc/arch-arm/syscalls/timerfd_gettime.S create mode 100644 libc/arch-arm/syscalls/timerfd_settime.S create mode 100644 libc/arch-mips/syscalls/timerfd_create.S create mode 100644 libc/arch-mips/syscalls/timerfd_gettime.S create mode 100644 libc/arch-mips/syscalls/timerfd_settime.S create mode 100644 libc/arch-x86/syscalls/timerfd_create.S create mode 100644 libc/arch-x86/syscalls/timerfd_gettime.S create mode 100644 libc/arch-x86/syscalls/timerfd_settime.S create mode 100644 libc/include/sys/timerfd.h create mode 100644 libc/kernel/common/linux/timerfd.h diff --git a/libc/SYSCALLS.TXT b/libc/SYSCALLS.TXT index 96cc9e6a9..ac04e51e4 100644 --- a/libc/SYSCALLS.TXT +++ b/libc/SYSCALLS.TXT @@ -221,6 +221,9 @@ int __timer_getoverrun:timer_getoverrun(timer_t) int __timer_delete:timer_delete(timer_t) 1 int utimes(const char*, const struct timeval tvp[2]) 1 int utimensat(int, const char *, const struct timespec times[2], int) 1 +int timerfd_create(clockid_t, int) 1 +int timerfd_settime(int, int, const struct itimerspec *, struct itimerspec *) 1 +int timerfd_gettime(int, struct itimerspec *) 1 # signals int sigaction(int, const struct sigaction *, struct sigaction *) 1 diff --git a/libc/arch-arm/syscalls.mk b/libc/arch-arm/syscalls.mk index 9eb51369e..3c8f20490 100644 --- a/libc/arch-arm/syscalls.mk +++ b/libc/arch-arm/syscalls.mk @@ -144,6 +144,9 @@ syscall_src += arch-arm/syscalls/__timer_getoverrun.S syscall_src += arch-arm/syscalls/__timer_delete.S syscall_src += arch-arm/syscalls/utimes.S syscall_src += arch-arm/syscalls/utimensat.S +syscall_src += arch-arm/syscalls/timerfd_create.S +syscall_src += arch-arm/syscalls/timerfd_settime.S +syscall_src += arch-arm/syscalls/timerfd_gettime.S syscall_src += arch-arm/syscalls/sigaction.S syscall_src += arch-arm/syscalls/sigprocmask.S syscall_src += arch-arm/syscalls/__sigsuspend.S diff --git a/libc/arch-arm/syscalls/timerfd_create.S b/libc/arch-arm/syscalls/timerfd_create.S new file mode 100644 index 000000000..7e3f16f02 --- /dev/null +++ b/libc/arch-arm/syscalls/timerfd_create.S @@ -0,0 +1,15 @@ +/* autogenerated by gensyscalls.py */ +#include +#include +#include + +ENTRY(timerfd_create) + mov ip, r7 + ldr r7, =__NR_timerfd_create + swi #0 + mov r7, ip + cmn r0, #(MAX_ERRNO + 1) + bxls lr + neg r0, r0 + b __set_errno +END(timerfd_create) diff --git a/libc/arch-arm/syscalls/timerfd_gettime.S b/libc/arch-arm/syscalls/timerfd_gettime.S new file mode 100644 index 000000000..2c3e2cf78 --- /dev/null +++ b/libc/arch-arm/syscalls/timerfd_gettime.S @@ -0,0 +1,15 @@ +/* autogenerated by gensyscalls.py */ +#include +#include +#include + +ENTRY(timerfd_gettime) + mov ip, r7 + ldr r7, =__NR_timerfd_gettime + swi #0 + mov r7, ip + cmn r0, #(MAX_ERRNO + 1) + bxls lr + neg r0, r0 + b __set_errno +END(timerfd_gettime) diff --git a/libc/arch-arm/syscalls/timerfd_settime.S b/libc/arch-arm/syscalls/timerfd_settime.S new file mode 100644 index 000000000..f7f0cf0e0 --- /dev/null +++ b/libc/arch-arm/syscalls/timerfd_settime.S @@ -0,0 +1,15 @@ +/* autogenerated by gensyscalls.py */ +#include +#include +#include + +ENTRY(timerfd_settime) + mov ip, r7 + ldr r7, =__NR_timerfd_settime + swi #0 + mov r7, ip + cmn r0, #(MAX_ERRNO + 1) + bxls lr + neg r0, r0 + b __set_errno +END(timerfd_settime) diff --git a/libc/arch-mips/syscalls.mk b/libc/arch-mips/syscalls.mk index 0b8eccd28..d3096bc78 100644 --- a/libc/arch-mips/syscalls.mk +++ b/libc/arch-mips/syscalls.mk @@ -147,6 +147,9 @@ syscall_src += arch-mips/syscalls/__timer_getoverrun.S syscall_src += arch-mips/syscalls/__timer_delete.S syscall_src += arch-mips/syscalls/utimes.S syscall_src += arch-mips/syscalls/utimensat.S +syscall_src += arch-mips/syscalls/timerfd_create.S +syscall_src += arch-mips/syscalls/timerfd_settime.S +syscall_src += arch-mips/syscalls/timerfd_gettime.S syscall_src += arch-mips/syscalls/sigaction.S syscall_src += arch-mips/syscalls/sigprocmask.S syscall_src += arch-mips/syscalls/__sigsuspend.S diff --git a/libc/arch-mips/syscalls/timerfd_create.S b/libc/arch-mips/syscalls/timerfd_create.S new file mode 100644 index 000000000..b5ac003cf --- /dev/null +++ b/libc/arch-mips/syscalls/timerfd_create.S @@ -0,0 +1,22 @@ +/* autogenerated by gensyscalls.py */ +#include + .text + .globl timerfd_create + .align 4 + .ent timerfd_create + +timerfd_create: + .set noreorder + .cpload $t9 + li $v0, __NR_timerfd_create + syscall + bnez $a3, 1f + move $a0, $v0 + j $ra + nop +1: + la $t9,__set_errno + j $t9 + nop + .set reorder + .end timerfd_create diff --git a/libc/arch-mips/syscalls/timerfd_gettime.S b/libc/arch-mips/syscalls/timerfd_gettime.S new file mode 100644 index 000000000..b1c21fff9 --- /dev/null +++ b/libc/arch-mips/syscalls/timerfd_gettime.S @@ -0,0 +1,22 @@ +/* autogenerated by gensyscalls.py */ +#include + .text + .globl timerfd_gettime + .align 4 + .ent timerfd_gettime + +timerfd_gettime: + .set noreorder + .cpload $t9 + li $v0, __NR_timerfd_gettime + syscall + bnez $a3, 1f + move $a0, $v0 + j $ra + nop +1: + la $t9,__set_errno + j $t9 + nop + .set reorder + .end timerfd_gettime diff --git a/libc/arch-mips/syscalls/timerfd_settime.S b/libc/arch-mips/syscalls/timerfd_settime.S new file mode 100644 index 000000000..f68819d69 --- /dev/null +++ b/libc/arch-mips/syscalls/timerfd_settime.S @@ -0,0 +1,22 @@ +/* autogenerated by gensyscalls.py */ +#include + .text + .globl timerfd_settime + .align 4 + .ent timerfd_settime + +timerfd_settime: + .set noreorder + .cpload $t9 + li $v0, __NR_timerfd_settime + syscall + bnez $a3, 1f + move $a0, $v0 + j $ra + nop +1: + la $t9,__set_errno + j $t9 + nop + .set reorder + .end timerfd_settime diff --git a/libc/arch-x86/syscalls.mk b/libc/arch-x86/syscalls.mk index b4ad564ed..31eb93008 100644 --- a/libc/arch-x86/syscalls.mk +++ b/libc/arch-x86/syscalls.mk @@ -148,6 +148,9 @@ syscall_src += arch-x86/syscalls/__timer_getoverrun.S syscall_src += arch-x86/syscalls/__timer_delete.S syscall_src += arch-x86/syscalls/utimes.S syscall_src += arch-x86/syscalls/utimensat.S +syscall_src += arch-x86/syscalls/timerfd_create.S +syscall_src += arch-x86/syscalls/timerfd_settime.S +syscall_src += arch-x86/syscalls/timerfd_gettime.S syscall_src += arch-x86/syscalls/sigaction.S syscall_src += arch-x86/syscalls/sigprocmask.S syscall_src += arch-x86/syscalls/__sigsuspend.S diff --git a/libc/arch-x86/syscalls/timerfd_create.S b/libc/arch-x86/syscalls/timerfd_create.S new file mode 100644 index 000000000..801f8a7f7 --- /dev/null +++ b/libc/arch-x86/syscalls/timerfd_create.S @@ -0,0 +1,24 @@ +/* autogenerated by gensyscalls.py */ +#include +#include +#include + +ENTRY(timerfd_create) + pushl %ebx + pushl %ecx + mov 12(%esp), %ebx + mov 16(%esp), %ecx + movl $__NR_timerfd_create, %eax + int $0x80 + cmpl $-MAX_ERRNO, %eax + jb 1f + negl %eax + pushl %eax + call __set_errno + addl $4, %esp + orl $-1, %eax +1: + popl %ecx + popl %ebx + ret +END(timerfd_create) diff --git a/libc/arch-x86/syscalls/timerfd_gettime.S b/libc/arch-x86/syscalls/timerfd_gettime.S new file mode 100644 index 000000000..fde17beaf --- /dev/null +++ b/libc/arch-x86/syscalls/timerfd_gettime.S @@ -0,0 +1,24 @@ +/* autogenerated by gensyscalls.py */ +#include +#include +#include + +ENTRY(timerfd_gettime) + pushl %ebx + pushl %ecx + mov 12(%esp), %ebx + mov 16(%esp), %ecx + movl $__NR_timerfd_gettime, %eax + int $0x80 + cmpl $-MAX_ERRNO, %eax + jb 1f + negl %eax + pushl %eax + call __set_errno + addl $4, %esp + orl $-1, %eax +1: + popl %ecx + popl %ebx + ret +END(timerfd_gettime) diff --git a/libc/arch-x86/syscalls/timerfd_settime.S b/libc/arch-x86/syscalls/timerfd_settime.S new file mode 100644 index 000000000..5a5f3e4e2 --- /dev/null +++ b/libc/arch-x86/syscalls/timerfd_settime.S @@ -0,0 +1,30 @@ +/* autogenerated by gensyscalls.py */ +#include +#include +#include + +ENTRY(timerfd_settime) + pushl %ebx + pushl %ecx + pushl %edx + pushl %esi + mov 20(%esp), %ebx + mov 24(%esp), %ecx + mov 28(%esp), %edx + mov 32(%esp), %esi + movl $__NR_timerfd_settime, %eax + int $0x80 + cmpl $-MAX_ERRNO, %eax + jb 1f + negl %eax + pushl %eax + call __set_errno + addl $4, %esp + orl $-1, %eax +1: + popl %esi + popl %edx + popl %ecx + popl %ebx + ret +END(timerfd_settime) diff --git a/libc/include/sys/timerfd.h b/libc/include/sys/timerfd.h new file mode 100644 index 000000000..0651f1ca6 --- /dev/null +++ b/libc/include/sys/timerfd.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _SYS_TIMERFD_H_ +#define _SYS_TIMERFD_H_ + +#include +#include +#include + +__BEGIN_DECLS + +extern int timerfd_create(clockid_t, int); +extern int timerfd_settime(int, int, const struct itimerspec*, + struct itimerspec*); +extern int timerfd_gettime(int, struct itimerspec*); + +__END_DECLS + +#endif /* _SYS_TIMERFD_H */ diff --git a/libc/kernel/common/linux/timerfd.h b/libc/kernel/common/linux/timerfd.h new file mode 100644 index 000000000..0165ebbb3 --- /dev/null +++ b/libc/kernel/common/linux/timerfd.h @@ -0,0 +1,31 @@ +/**************************************************************************** + **************************************************************************** + *** + *** This header was automatically generated from a Linux kernel header + *** of the same name, to make information necessary for userspace to + *** call into the kernel available to libc. It contains only constants, + *** structures, and macros generated from the original header, and thus, + *** contains no copyrightable information. + *** + *** To edit the content of this header, modify the corresponding + *** source file (e.g. under external/kernel-headers/original/) then + *** run bionic/libc/kernel/tools/update_all.py + *** + *** Any manual change here will be lost the next time this script will + *** be run. You've been warned! + *** + **************************************************************************** + ****************************************************************************/ +#ifndef _LINUX_TIMERFD_H +#define _LINUX_TIMERFD_H +#include +#define TFD_TIMER_ABSTIME (1 << 0) +/* WARNING: DO NOT EDIT, AUTO-GENERATED CODE - SEE TOP FOR INSTRUCTIONS */ +#define TFD_TIMER_CANCEL_ON_SET (1 << 1) +#define TFD_CLOEXEC O_CLOEXEC +#define TFD_NONBLOCK O_NONBLOCK +#define TFD_SHARED_FCNTL_FLAGS (TFD_CLOEXEC | TFD_NONBLOCK) +/* WARNING: DO NOT EDIT, AUTO-GENERATED CODE - SEE TOP FOR INSTRUCTIONS */ +#define TFD_CREATE_FLAGS TFD_SHARED_FCNTL_FLAGS +#define TFD_SETTIME_FLAGS (TFD_TIMER_ABSTIME | TFD_TIMER_CANCEL_ON_SET) +#endif From d1ad4f6dab06189d4d3dcfa19ae4bc301481eb3f Mon Sep 17 00:00:00 2001 From: Todd Poynor Date: Thu, 9 May 2013 14:38:39 -0700 Subject: [PATCH 12/77] epoll: add EPOLLRDHUP, EPOLLWAKEUP, and EPOLLONESHOT events Change-Id: I4a579fc4a74adaee57911b3974ec2f93d9321e8b --- libc/include/sys/epoll.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libc/include/sys/epoll.h b/libc/include/sys/epoll.h index decdb4649..38739aa13 100644 --- a/libc/include/sys/epoll.h +++ b/libc/include/sys/epoll.h @@ -42,6 +42,9 @@ __BEGIN_DECLS #define EPOLLWRNORM 0x00000100 #define EPOLLWRBAND 0x00000200 #define EPOLLMSG 0x00000400 +#define EPOLLRDHUP 0x00002000 +#define EPOLLWAKEUP 0x20000000 +#define EPOLLONESHOT 0x40000000 #define EPOLLET 0x80000000 #define EPOLL_CTL_ADD 1 From c702a904679a36511bead29c51eeac15d81f4fd2 Mon Sep 17 00:00:00 2001 From: Rom Lemarchand Date: Fri, 17 May 2013 17:08:41 -0700 Subject: [PATCH 13/77] mmap: Reinstate passing MADV_MERGEABLE on private anonymous maps Reinstate mmap calling madvise(MADV_MERGEABLE) removed in 635df850e5037be5093f64a87ec2e0a23bf7a50b Change-Id: I290bc5ac6bb32735a3f61dd21e2fce1dfb6dfd4b Signed-off-by: Rom Lemarchand --- libc/bionic/mmap.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/libc/bionic/mmap.c b/libc/bionic/mmap.c index 40a65385b..e097086cb 100644 --- a/libc/bionic/mmap.c +++ b/libc/bionic/mmap.c @@ -34,10 +34,17 @@ extern void* __mmap2(void*, size_t, int, int, int, size_t); #define MMAP2_SHIFT 12 void* mmap(void *addr, size_t size, int prot, int flags, int fd, long offset) { + void *ret; + if (offset & ((1UL << MMAP2_SHIFT)-1)) { errno = EINVAL; return MAP_FAILED; } - return __mmap2(addr, size, prot, flags, fd, (size_t)offset >> MMAP2_SHIFT); + ret = __mmap2(addr, size, prot, flags, fd, (size_t)offset >> MMAP2_SHIFT); + + if (ret && (flags & (MAP_PRIVATE | MAP_ANONYMOUS))) + madvise(ret, size, MADV_MERGEABLE); + + return ret; } From 9e1905794b4ecd8f7b87d8e4e2f954c8cfc6beda Mon Sep 17 00:00:00 2001 From: Ben Cheng Date: Fri, 31 May 2013 14:25:48 -0700 Subject: [PATCH 14/77] Use bl instead of blx to support interworking properly. BUG: 9227177 Change-Id: I742c2f2ecbe332f9c9743e3f4bde8de791a1d289 --- libc/arch-arm/bionic/abort_arm.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/arch-arm/bionic/abort_arm.S b/libc/arch-arm/bionic/abort_arm.S index 51b0871fa..e1ab86b6e 100644 --- a/libc/arch-arm/bionic/abort_arm.S +++ b/libc/arch-arm/bionic/abort_arm.S @@ -38,5 +38,5 @@ ENTRY(abort) .save {r3, r14} stmfd sp!, {r3, r14} - blx PIC_SYM(_C_LABEL(__libc_android_abort), PLT) + bl PIC_SYM(_C_LABEL(__libc_android_abort), PLT) END(abort) From 0c9bb49a9d44e9f1ae98cd56c2dfc24fbb40b53e Mon Sep 17 00:00:00 2001 From: Chad Brubaker Date: Thu, 30 May 2013 13:37:02 -0700 Subject: [PATCH 15/77] Add per UID interface support to resolv cache Add methods to attach/detach UID ranges to a specific dns cache/interface. This mirrors the already existing code for attaching specific processes to specific interfaces but will be used to push all processes from a given user to a specific cache/interface. Change-Id: Ic24391e92d3ca46fcb46cc4fc53e13984dec40b3 --- libc/netbsd/resolv/res_cache.c | 144 +++++++++++++++++++++++++++++++++ libc/private/resolv_iface.h | 18 +++++ 2 files changed, 162 insertions(+) diff --git a/libc/netbsd/resolv/res_cache.c b/libc/netbsd/resolv/res_cache.c index 357620263..b2d5f1b89 100644 --- a/libc/netbsd/resolv/res_cache.c +++ b/libc/netbsd/resolv/res_cache.c @@ -1258,6 +1258,12 @@ typedef struct resolv_pidiface_info { char ifname[IF_NAMESIZE + 1]; struct resolv_pidiface_info* next; } PidIfaceInfo; +typedef struct resolv_uidiface_info { + int low; + int high; + char ifname[IF_NAMESIZE + 1]; + struct resolv_uidiface_info* next; +} UidIfaceInfo; #define HTABLE_VALID(x) ((x) != NULL && (x) != HTABLE_DELETED) @@ -1796,6 +1802,9 @@ static struct resolv_cache_info _res_cache_list; // List of pid iface pairs static struct resolv_pidiface_info _res_pidiface_list; +// List of uid iface pairs +static struct resolv_uidiface_info _res_uidiface_list; + // name of the current default inteface static char _res_default_ifname[IF_NAMESIZE + 1]; @@ -1805,6 +1814,9 @@ static pthread_mutex_t _res_cache_list_lock; // lock protecting the _res_pid_iface_list static pthread_mutex_t _res_pidiface_list_lock; +// lock protecting the _res_uidiface_list +static pthread_mutex_t _res_uidiface_list_lock; + /* lookup the default interface name */ static char *_get_default_iface_locked(); /* find the first cache that has an associated interface and return the name of the interface */ @@ -1839,6 +1851,13 @@ static void _remove_pidiface_info_locked(int pid); /* get a resolv_pidiface_info structure from _res_pidiface_list with a certain pid */ static struct resolv_pidiface_info* _get_pid_iface_info_locked(int pid); +/* remove a resolv_pidiface_info structure from _res_uidiface_list */ +static int _remove_uidiface_info_locked(int low, int high); +/* check if a range [low,high] overlaps with any already existing ranges in the uid=>iface map*/ +static int _resolv_check_uid_range_overlap_locked(int low, int high); +/* get a resolv_uidiface_info structure from _res_uidiface_list with a certain uid */ +static struct resolv_uidiface_info* _get_uid_iface_info_locked(int uid); + static void _res_cache_init(void) { @@ -1852,8 +1871,10 @@ _res_cache_init(void) memset(&_res_default_ifname, 0, sizeof(_res_default_ifname)); memset(&_res_cache_list, 0, sizeof(_res_cache_list)); memset(&_res_pidiface_list, 0, sizeof(_res_pidiface_list)); + memset(&_res_uidiface_list, 0, sizeof(_res_uidiface_list)); pthread_mutex_init(&_res_cache_list_lock, NULL); pthread_mutex_init(&_res_pidiface_list_lock, NULL); + pthread_mutex_init(&_res_uidiface_list_lock, NULL); } struct resolv_cache* @@ -2411,6 +2432,129 @@ _resolv_get_pids_associated_interface(int pid, char* buff, int buffLen) return len; } +static int +_remove_uidiface_info_locked(int low, int high) { + struct resolv_uidiface_info* result = &_res_uidiface_list.next; + struct resolv_uidiface_info* prev = NULL; + + while (result != NULL && result->low != low && result->high != high) { + prev = result; + result = result->next; + } + if (prev != NULL && result != NULL) { + prev->next = result->next; + free(result); + return 0; + } + errno = EINVAL; + return -1; +} + +static struct resolv_uidiface_info* +_get_uid_iface_info_locked(int uid) +{ + struct resolv_uidiface_info* result = &_res_uidiface_list.next; + while (result != NULL && !(result->low <= uid && result->high >= uid)) { + result = result->next; + } + + return result; +} + +static int +_resolv_check_uid_range_overlap_locked(int low, int high) +{ + struct resolv_uidiface_info* cur = &_res_uidiface_list.next; + while (cur != NULL) { + if (cur->low <= high && cur->high >= low) { + return -1; + } + cur = cur->next; + } + return 0; +} + +int +_resolv_set_iface_for_uid_range(const char* ifname, int low, int high) +{ + int rv = 0; + struct resolv_uidiface_info* uidiface_info; + // make sure the uid iface list is created + pthread_once(&_res_cache_once, _res_cache_init); + pthread_mutex_lock(&_res_uidiface_list_lock); + //check that we aren't adding an overlapping range + if (!_resolv_check_uid_range_overlap_locked(low,high)) { + uidiface_info = calloc(sizeof(*uidiface_info), 1); + if (uidiface_info) { + uidiface_info->low = low; + uidiface_info->high = high; + int len = sizeof(uidiface_info->ifname); + strncpy(uidiface_info->ifname, ifname, len - 1); + uidiface_info->ifname[len - 1] = '\0'; + + uidiface_info->next = _res_uidiface_list.next; + _res_uidiface_list.next = uidiface_info; + + XLOG("_resolv_set_iface_for_uid_range: [%d,%d], iface %s\n", low, high, ifname); + } else { + XLOG("_resolv_set_iface_for_uid_range failing calloc\n"); + rv = -1; + errno = EINVAL; + } + } else { + XLOG("_resolv_set_iface_for_uid_range range [%d,%d] overlaps\n", low, high); + rv = -1; + errno = EINVAL; + } + + pthread_mutex_unlock(&_res_uidiface_list_lock); + return rv; +} + +int +_resolv_clear_iface_for_uid_range(int low, int high) +{ + pthread_once(&_res_cache_once, _res_cache_init); + pthread_mutex_lock(&_res_uidiface_list_lock); + + int rv = _remove_uidiface_info_locked(low, high); + + XLOG("_resolv_clear_iface_for_uid_range: low %d high %d\n", low, high); + + pthread_mutex_unlock(&_res_uidiface_list_lock); + + return rv; +} + +int +_resolv_get_uids_associated_interface(int uid, char* buff, int buffLen) +{ + int len = 0; + + if (!buff) { + return -1; + } + + pthread_once(&_res_cache_once, _res_cache_init); + pthread_mutex_lock(&_res_uidiface_list_lock); + + struct resolv_uidiface_info* uidiface_info = _get_uid_iface_info_locked(uid); + buff[0] = '\0'; + if (uidiface_info) { + len = strlen(uidiface_info->ifname); + if (len < buffLen) { + strncpy(buff, uidiface_info->ifname, len); + buff[len] = '\0'; + } + } + + XLOG("_resolv_get_uids_associated_interface buff: %s\n", buff); + + pthread_mutex_unlock(&_res_uidiface_list_lock); + + return len; +} + size_t _resolv_get_default_iface(char* buff, size_t buffLen) { diff --git a/libc/private/resolv_iface.h b/libc/private/resolv_iface.h index 789d99719..6be97c1fd 100644 --- a/libc/private/resolv_iface.h +++ b/libc/private/resolv_iface.h @@ -75,6 +75,24 @@ extern void _resolv_clear_iface_for_pid(int pid); * buffLen Length of buff. An interface is at most IF_NAMESIZE in length */ extern int _resolv_get_pids_associated_interface(int pid, char* buff, int buffLen); + +/** set a uid range to use the name servers of the specified interface + * If [low,high] overlaps with an already existing rule -1 is returned */ +extern int _resolv_set_iface_for_uid_range(const char* ifname, int low, int high); + +/* clear a uid range from being associated with an interface + * If the range given is not mapped -1 is returned. */ +extern int _resolv_clear_iface_for_uid_range(int low, int high); + +/** Gets the name of the interface to which the uid is attached. + * On error, -1 is returned. + * If no interface is found, 0 is returned and buff is set to empty ('\0'). + * If an interface is found, the name is copied to buff and the length of the name is returned. + * Arguments: uid The uid to find an interface for + * buff A buffer to copy the result to + * buffLen Length of buff. An interface is at most IF_NAMESIZE in length */ +extern int _resolv_get_uids_associated_interface(int uid, char* buff, int buffLen); + #endif /* _BIONIC_RESOLV_IFACE_FUNCTIONS_DECLARED */ __END_DECLS From 3d594c258045783fc9e1956ce7a4d91e302f011e Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Tue, 21 May 2013 17:48:01 -0700 Subject: [PATCH 16/77] Implement malloc_usable_size for debug impls. - Implemented chk_memalign. - Fixed a few bugs in leak_memalign. - Implemented {leak,fill,check,qemu}_malloc_usable_size. - Make malloc_usable_size update at run time. - Add malloc_test.cpp as a small set of tests for the malloc debug routines. - Fix the qemu routines since it's been broken since it moved to C++. - Add support for the %u format to the out_vformat in libc_logging.cpp. This is used by the emulator code. Tested using the bionic-unit-tests with setprop libc.debug.malloc set to 1, 5, and 10. I tested as much as possible on the emulator, but tracing doesn't appear to be working properly. Bug: 6143477 Change-Id: Ieba99b58c2228c88d80afd264501004a4dadd212 --- libc/bionic/libc_logging.cpp | 2 +- libc/bionic/malloc_debug_check.cpp | 139 +++++++++++----- libc/bionic/malloc_debug_common.cpp | 105 ++++++------- libc/bionic/malloc_debug_common.h | 7 + libc/bionic/malloc_debug_leak.cpp | 107 +++++++++---- libc/bionic/malloc_debug_qemu.cpp | 53 +++++-- tests/Android.mk | 1 + tests/malloc_test.cpp | 235 ++++++++++++++++++++++++++++ 8 files changed, 505 insertions(+), 144 deletions(-) create mode 100644 tests/malloc_test.cpp diff --git a/libc/bionic/libc_logging.cpp b/libc/bionic/libc_logging.cpp index 8de1192be..74e599c45 100644 --- a/libc/bionic/libc_logging.cpp +++ b/libc/bionic/libc_logging.cpp @@ -348,7 +348,7 @@ static void out_vformat(Out& o, const char* format, va_list args) { buffer[0] = '0'; buffer[1] = 'x'; format_integer(buffer + 2, sizeof(buffer) - 2, value, 'x'); - } else if (c == 'd' || c == 'i' || c == 'o' || c == 'x' || c == 'X') { + } else if (c == 'd' || c == 'i' || c == 'o' || c == 'u' || c == 'x' || c == 'X') { /* integers - first read value from stack */ uint64_t value; int is_signed = (c == 'd' || c == 'i' || c == 'o'); diff --git a/libc/bionic/malloc_debug_check.cpp b/libc/bionic/malloc_debug_check.cpp index 91cf28799..11a6ec192 100644 --- a/libc/bionic/malloc_debug_check.cpp +++ b/libc/bionic/malloc_debug_check.cpp @@ -74,6 +74,10 @@ static void log_message(const char* format, ...) { struct hdr_t { uint32_t tag; + void* base; // Always points to the memory allocated using dlmalloc. + // For memory allocated in chk_memalign, this value will + // not be the same as the location of the start of this + // structure. hdr_t* prev; hdr_t* next; uintptr_t bt[MAX_BACKTRACE_DEPTH]; @@ -82,7 +86,7 @@ struct hdr_t { int freed_bt_depth; size_t size; char front_guard[FRONT_GUARD_LEN]; -} __attribute__((packed)); +} __attribute__((packed, aligned(MALLOC_ALIGNMENT))); struct ftr_t { char rear_guard[REAR_GUARD_LEN]; @@ -100,21 +104,26 @@ static inline hdr_t* meta(void* user) { return reinterpret_cast(user) - 1; } +static inline const hdr_t* const_meta(const void* user) { + return reinterpret_cast(user) - 1; +} + + static unsigned gAllocatedBlockCount; -static hdr_t *tail; -static hdr_t *head; +static hdr_t* tail; +static hdr_t* head; static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; static unsigned backlog_num; -static hdr_t *backlog_tail; -static hdr_t *backlog_head; +static hdr_t* backlog_tail; +static hdr_t* backlog_head; static pthread_mutex_t backlog_lock = PTHREAD_MUTEX_INITIALIZER; -static inline void init_front_guard(hdr_t *hdr) { +static inline void init_front_guard(hdr_t* hdr) { memset(hdr->front_guard, FRONT_GUARD, FRONT_GUARD_LEN); } -static inline bool is_front_guard_valid(hdr_t *hdr) { +static inline bool is_front_guard_valid(hdr_t* hdr) { for (size_t i = 0; i < FRONT_GUARD_LEN; i++) { if (hdr->front_guard[i] != FRONT_GUARD) { return 0; @@ -123,12 +132,12 @@ static inline bool is_front_guard_valid(hdr_t *hdr) { return 1; } -static inline void init_rear_guard(hdr_t *hdr) { +static inline void init_rear_guard(hdr_t* hdr) { ftr_t* ftr = to_ftr(hdr); memset(ftr->rear_guard, REAR_GUARD, REAR_GUARD_LEN); } -static inline bool is_rear_guard_valid(hdr_t *hdr) { +static inline bool is_rear_guard_valid(hdr_t* hdr) { unsigned i; int valid = 1; int first_mismatch = -1; @@ -149,7 +158,7 @@ static inline bool is_rear_guard_valid(hdr_t *hdr) { return valid; } -static inline void add_locked(hdr_t *hdr, hdr_t **tail, hdr_t **head) { +static inline void add_locked(hdr_t* hdr, hdr_t** tail, hdr_t** head) { hdr->prev = NULL; hdr->next = *head; if (*head) @@ -159,7 +168,7 @@ static inline void add_locked(hdr_t *hdr, hdr_t **tail, hdr_t **head) { *head = hdr; } -static inline int del_locked(hdr_t *hdr, hdr_t **tail, hdr_t **head) { +static inline int del_locked(hdr_t* hdr, hdr_t** tail, hdr_t** head) { if (hdr->prev) { hdr->prev->next = hdr->next; } else { @@ -173,7 +182,7 @@ static inline int del_locked(hdr_t *hdr, hdr_t **tail, hdr_t **head) { return 0; } -static inline void add(hdr_t *hdr, size_t size) { +static inline void add(hdr_t* hdr, size_t size) { ScopedPthreadMutexLocker locker(&lock); hdr->tag = ALLOCATION_TAG; hdr->size = size; @@ -183,7 +192,7 @@ static inline void add(hdr_t *hdr, size_t size) { add_locked(hdr, &tail, &head); } -static inline int del(hdr_t *hdr) { +static inline int del(hdr_t* hdr) { if (hdr->tag != ALLOCATION_TAG) { return -1; } @@ -194,13 +203,13 @@ static inline int del(hdr_t *hdr) { return 0; } -static inline void poison(hdr_t *hdr) { +static inline void poison(hdr_t* hdr) { memset(user(hdr), FREE_POISON, hdr->size); } -static int was_used_after_free(hdr_t *hdr) { +static int was_used_after_free(hdr_t* hdr) { unsigned i; - const char *data = (const char *)user(hdr); + const char* data = reinterpret_cast(user(hdr)); for (i = 0; i < hdr->size; i++) if (data[i] != FREE_POISON) return 1; @@ -208,7 +217,7 @@ static int was_used_after_free(hdr_t *hdr) { } /* returns 1 if valid, *safe == 1 if safe to dump stack */ -static inline int check_guards(hdr_t *hdr, int *safe) { +static inline int check_guards(hdr_t* hdr, int* safe) { *safe = 1; if (!is_front_guard_valid(hdr)) { if (hdr->front_guard[0] == FRONT_GUARD) { @@ -233,7 +242,7 @@ static inline int check_guards(hdr_t *hdr, int *safe) { } /* returns 1 if valid, *safe == 1 if safe to dump stack */ -static inline int check_allocation_locked(hdr_t *hdr, int *safe) { +static inline int check_allocation_locked(hdr_t* hdr, int* safe) { int valid = 1; *safe = 1; @@ -270,9 +279,9 @@ static inline int check_allocation_locked(hdr_t *hdr, int *safe) { return valid; } -static inline int del_and_check_locked(hdr_t *hdr, - hdr_t **tail, hdr_t **head, unsigned *cnt, - int *safe) { +static inline int del_and_check_locked(hdr_t* hdr, + hdr_t** tail, hdr_t** head, unsigned* cnt, + int* safe) { int valid = check_allocation_locked(hdr, safe); if (safe) { (*cnt)--; @@ -281,7 +290,7 @@ static inline int del_and_check_locked(hdr_t *hdr, return valid; } -static inline void del_from_backlog_locked(hdr_t *hdr) { +static inline void del_from_backlog_locked(hdr_t* hdr) { int safe; del_and_check_locked(hdr, &backlog_tail, &backlog_head, &backlog_num, @@ -289,17 +298,17 @@ static inline void del_from_backlog_locked(hdr_t *hdr) { hdr->tag = 0; /* clear the tag */ } -static inline void del_from_backlog(hdr_t *hdr) { +static inline void del_from_backlog(hdr_t* hdr) { ScopedPthreadMutexLocker locker(&backlog_lock); del_from_backlog_locked(hdr); } -static inline int del_leak(hdr_t *hdr, int *safe) { +static inline int del_leak(hdr_t* hdr, int* safe) { ScopedPthreadMutexLocker locker(&lock); return del_and_check_locked(hdr, &tail, &head, &gAllocatedBlockCount, safe); } -static inline void add_to_backlog(hdr_t *hdr) { +static inline void add_to_backlog(hdr_t* hdr) { ScopedPthreadMutexLocker locker(&backlog_lock); hdr->tag = BACKLOG_TAG; backlog_num++; @@ -307,9 +316,9 @@ static inline void add_to_backlog(hdr_t *hdr) { poison(hdr); /* If we've exceeded the maximum backlog, clear it up */ while (backlog_num > gMallocDebugBacklog) { - hdr_t *gone = backlog_tail; + hdr_t* gone = backlog_tail; del_from_backlog_locked(gone); - dlfree(gone); + dlfree(gone->base); } } @@ -318,6 +327,7 @@ extern "C" void* chk_malloc(size_t size) { hdr_t* hdr = static_cast(dlmalloc(sizeof(hdr_t) + size + sizeof(ftr_t))); if (hdr) { + hdr->base = hdr; hdr->bt_depth = get_backtrace(hdr->bt, MAX_BACKTRACE_DEPTH); add(hdr, size); return user(hdr); @@ -325,13 +335,44 @@ extern "C" void* chk_malloc(size_t size) { return NULL; } -extern "C" void* chk_memalign(size_t, size_t bytes) { -// log_message("%s: %s\n", __FILE__, __FUNCTION__); - // XXX: it's better to use malloc, than being wrong - return chk_malloc(bytes); +extern "C" void* chk_memalign(size_t alignment, size_t bytes) { + if (alignment <= MALLOC_ALIGNMENT) { + return chk_malloc(bytes); + } + + // Make the alignment a power of two. + if (alignment & (alignment-1)) { + alignment = 1L << (31 - __builtin_clz(alignment)); + } + + // here, alignment is at least MALLOC_ALIGNMENT<<1 bytes + // we will align by at least MALLOC_ALIGNMENT bytes + // and at most alignment-MALLOC_ALIGNMENT bytes + size_t size = (alignment-MALLOC_ALIGNMENT) + bytes; + if (size < bytes) { // Overflow. + return NULL; + } + + void* base = dlmalloc(sizeof(hdr_t) + size + sizeof(ftr_t)); + if (base != NULL) { + // Check that the actual pointer that will be returned is aligned + // properly. + uintptr_t ptr = reinterpret_cast(user(reinterpret_cast(base))); + if ((ptr % alignment) != 0) { + // Align the pointer. + ptr += ((-ptr) % alignment); + } + + hdr_t* hdr = meta(reinterpret_cast(ptr)); + hdr->base = base; + hdr->bt_depth = get_backtrace(hdr->bt, MAX_BACKTRACE_DEPTH); + add(hdr, bytes); + return user(hdr); + } + return base; } -extern "C" void chk_free(void *ptr) { +extern "C" void chk_free(void* ptr) { // log_message("%s: %s\n", __FILE__, __FUNCTION__); if (!ptr) /* ignore free(NULL) */ @@ -366,7 +407,7 @@ extern "C" void chk_free(void *ptr) { } } -extern "C" void *chk_realloc(void *ptr, size_t size) { +extern "C" void* chk_realloc(void* ptr, size_t size) { // log_message("%s: %s\n", __FILE__, __FUNCTION__); if (!ptr) { @@ -414,8 +455,23 @@ extern "C" void *chk_realloc(void *ptr, size_t size) { } } - hdr = static_cast(dlrealloc(hdr, sizeof(hdr_t) + size + sizeof(ftr_t))); + if (hdr->base != hdr) { + // An allocation from memalign, so create another allocation and + // copy the data out. + void* newMem = dlmalloc(sizeof(hdr_t) + size + sizeof(ftr_t)); + if (newMem) { + memcpy(newMem, hdr, sizeof(hdr_t) + hdr->size); + dlfree(hdr->base); + hdr = static_cast(newMem); + } else { + dlfree(hdr->base); + hdr = NULL; + } + } else { + hdr = static_cast(dlrealloc(hdr, sizeof(hdr_t) + size + sizeof(ftr_t))); + } if (hdr) { + hdr->base = hdr; hdr->bt_depth = get_backtrace(hdr->bt, MAX_BACKTRACE_DEPTH); add(hdr, size); return user(hdr); @@ -424,11 +480,12 @@ extern "C" void *chk_realloc(void *ptr, size_t size) { return NULL; } -extern "C" void *chk_calloc(int nmemb, size_t size) { +extern "C" void* chk_calloc(int nmemb, size_t size) { // log_message("%s: %s\n", __FILE__, __FUNCTION__); size_t total_size = nmemb * size; hdr_t* hdr = static_cast(dlcalloc(1, sizeof(hdr_t) + total_size + sizeof(ftr_t))); if (hdr) { + hdr->base = hdr; hdr->bt_depth = get_backtrace(hdr->bt, MAX_BACKTRACE_DEPTH); add(hdr, total_size); return user(hdr); @@ -436,6 +493,18 @@ extern "C" void *chk_calloc(int nmemb, size_t size) { return NULL; } +extern "C" size_t chk_malloc_usable_size(const void* ptr) { + // dlmalloc_usable_size returns 0 for NULL and unknown blocks. + if (ptr == NULL) + return 0; + + const hdr_t* hdr = const_meta(ptr); + + // The sentinel tail is written just after the request block bytes + // so there is no extra room we can report here. + return hdr->size; +} + static void ReportMemoryLeaks() { // We only track leaks at level 10. if (gMallocDebugLevel != 10) { diff --git a/libc/bionic/malloc_debug_common.cpp b/libc/bionic/malloc_debug_common.cpp index 9cc84c34f..ccceb1441 100644 --- a/libc/bionic/malloc_debug_common.cpp +++ b/libc/bionic/malloc_debug_common.cpp @@ -190,10 +190,6 @@ extern "C" struct mallinfo mallinfo() { return dlmallinfo(); } -extern "C" size_t malloc_usable_size(const void* mem) { - return dlmalloc_usable_size(mem); -} - extern "C" void* valloc(size_t bytes) { return dlvalloc(bytes); } @@ -215,8 +211,9 @@ extern "C" int posix_memalign(void** memptr, size_t alignment, size_t size) { /* Table for dispatching malloc calls, initialized with default dispatchers. */ extern const MallocDebug __libc_malloc_default_dispatch; -const MallocDebug __libc_malloc_default_dispatch __attribute__((aligned(32))) = { - dlmalloc, dlfree, dlcalloc, dlrealloc, dlmemalign +const MallocDebug __libc_malloc_default_dispatch __attribute__((aligned(32))) = +{ + dlmalloc, dlfree, dlcalloc, dlrealloc, dlmemalign, dlmalloc_usable_size, }; /* Selector of dispatch table to use for dispatching malloc calls. */ @@ -242,6 +239,10 @@ extern "C" void* memalign(size_t alignment, size_t bytes) { return __libc_malloc_dispatch->memalign(alignment, bytes); } +extern "C" size_t malloc_usable_size(const void* mem) { + return __libc_malloc_dispatch->malloc_usable_size(mem); +} + /* We implement malloc debugging only in libc.so, so code below * must be excluded if we compile this file for static libc.a */ @@ -253,7 +254,7 @@ extern "C" void* memalign(size_t alignment, size_t bytes) { /* Table for dispatching malloc calls, depending on environment. */ static MallocDebug gMallocUse __attribute__((aligned(32))) = { - dlmalloc, dlfree, dlcalloc, dlrealloc, dlmemalign + dlmalloc, dlfree, dlcalloc, dlrealloc, dlmemalign, dlmalloc_usable_size }; extern const char* __progname; @@ -276,15 +277,10 @@ extern const char* __progname; * Actual functionality for debug levels 1-10 is implemented in * libc_malloc_debug_leak.so, while functionality for emultor's instrumented * allocations is implemented in libc_malloc_debug_qemu.so and can be run inside - * the emulator only. + * the emulator only. */ static void* libc_malloc_impl_handle = NULL; -// This must match the alignment used by dlmalloc. -#ifndef MALLOC_ALIGNMENT -#define MALLOC_ALIGNMENT ((size_t)(2 * sizeof(void *))) -#endif - /* This variable is set to the value of property libc.debug.malloc.backlog, * when the value of libc.debug.malloc = 10. It determines the size of the * backlog we use to detect multiple frees. If the property is not set, the @@ -296,41 +292,26 @@ unsigned int gMallocDebugBacklog; /* The value of libc.debug.malloc. */ int gMallocDebugLevel; -static void InitMalloc(MallocDebug* table, const char* prefix) { - __libc_format_log(ANDROID_LOG_INFO, "libc", "%s: using libc.debug.malloc %d (%s)\n", - __progname, gMallocDebugLevel, prefix); +template +void InitMallocFunction(void* malloc_impl_handler, FunctionType* func, const char* prefix, const char* suffix) { + char symbol[128]; + snprintf(symbol, sizeof(symbol), "%s_%s", prefix, suffix); + *func = reinterpret_cast(dlsym(malloc_impl_handler, symbol)); + if (*func == NULL) { + error_log("%s: dlsym(\"%s\") failed", __progname, symbol); + } +} - char symbol[128]; +static void InitMalloc(void* malloc_impl_handler, MallocDebug* table, const char* prefix) { + __libc_format_log(ANDROID_LOG_INFO, "libc", "%s: using libc.debug.malloc %d (%s)\n", + __progname, gMallocDebugLevel, prefix); - snprintf(symbol, sizeof(symbol), "%s_malloc", prefix); - table->malloc = reinterpret_cast(dlsym(libc_malloc_impl_handle, symbol)); - if (table->malloc == NULL) { - error_log("%s: dlsym(\"%s\") failed", __progname, symbol); - } - - snprintf(symbol, sizeof(symbol), "%s_free", prefix); - table->free = reinterpret_cast(dlsym(libc_malloc_impl_handle, symbol)); - if (table->free == NULL) { - error_log("%s: dlsym(\"%s\") failed", __progname, symbol); - } - - snprintf(symbol, sizeof(symbol), "%s_calloc", prefix); - table->calloc = reinterpret_cast(dlsym(libc_malloc_impl_handle, symbol)); - if (table->calloc == NULL) { - error_log("%s: dlsym(\"%s\") failed", __progname, symbol); - } - - snprintf(symbol, sizeof(symbol), "%s_realloc", prefix); - table->realloc = reinterpret_cast(dlsym(libc_malloc_impl_handle, symbol)); - if (table->realloc == NULL) { - error_log("%s: dlsym(\"%s\") failed", __progname, symbol); - } - - snprintf(symbol, sizeof(symbol), "%s_memalign", prefix); - table->memalign = reinterpret_cast(dlsym(libc_malloc_impl_handle, symbol)); - if (table->memalign == NULL) { - error_log("%s: dlsym(\"%s\") failed", __progname, symbol); - } + InitMallocFunction(malloc_impl_handler, &table->malloc, prefix, "malloc"); + InitMallocFunction(malloc_impl_handler, &table->free, prefix, "free"); + InitMallocFunction(malloc_impl_handler, &table->calloc, prefix, "calloc"); + InitMallocFunction(malloc_impl_handler, &table->realloc, prefix, "realloc"); + InitMallocFunction(malloc_impl_handler, &table->memalign, prefix, "memalign"); + InitMallocFunction(malloc_impl_handler, &table->malloc_usable_size, prefix, "malloc_usable_size"); } /* Initializes memory allocation framework once per process. */ @@ -422,24 +403,24 @@ static void malloc_init_impl() { } // Load .so that implements the required malloc debugging functionality. - libc_malloc_impl_handle = dlopen(so_name, RTLD_LAZY); - if (libc_malloc_impl_handle == NULL) { + void* malloc_impl_handle = dlopen(so_name, RTLD_LAZY); + if (malloc_impl_handle == NULL) { error_log("%s: Missing module %s required for malloc debug level %d: %s", __progname, so_name, gMallocDebugLevel, dlerror()); return; } // Initialize malloc debugging in the loaded module. - malloc_debug_initialize = reinterpret_cast(dlsym(libc_malloc_impl_handle, + malloc_debug_initialize = reinterpret_cast(dlsym(malloc_impl_handle, "malloc_debug_initialize")); if (malloc_debug_initialize == NULL) { error_log("%s: Initialization routine is not found in %s\n", __progname, so_name); - dlclose(libc_malloc_impl_handle); + dlclose(malloc_impl_handle); return; } if (malloc_debug_initialize() == -1) { - dlclose(libc_malloc_impl_handle); + dlclose(malloc_impl_handle); return; } @@ -447,34 +428,35 @@ static void malloc_init_impl() { // For memory checker we need to do extra initialization. typedef int (*MemCheckInit)(int, const char*); MemCheckInit memcheck_initialize = - reinterpret_cast(dlsym(libc_malloc_impl_handle, + reinterpret_cast(dlsym(malloc_impl_handle, "memcheck_initialize")); if (memcheck_initialize == NULL) { error_log("%s: memcheck_initialize routine is not found in %s\n", __progname, so_name); - dlclose(libc_malloc_impl_handle); + dlclose(malloc_impl_handle); return; } if (memcheck_initialize(MALLOC_ALIGNMENT, memcheck_tracing)) { - dlclose(libc_malloc_impl_handle); + dlclose(malloc_impl_handle); return; } } + // Initialize malloc dispatch table with appropriate routines. switch (gMallocDebugLevel) { case 1: - InitMalloc(&gMallocUse, "leak"); + InitMalloc(malloc_impl_handle, &gMallocUse, "leak"); break; case 5: - InitMalloc(&gMallocUse, "fill"); + InitMalloc(malloc_impl_handle, &gMallocUse, "fill"); break; case 10: - InitMalloc(&gMallocUse, "chk"); + InitMalloc(malloc_impl_handle, &gMallocUse, "chk"); break; case 20: - InitMalloc(&gMallocUse, "qemu_instrumented"); + InitMalloc(malloc_impl_handle, &gMallocUse, "qemu_instrumented"); break; default: break; @@ -485,13 +467,14 @@ static void malloc_init_impl() { (gMallocUse.free == NULL) || (gMallocUse.calloc == NULL) || (gMallocUse.realloc == NULL) || - (gMallocUse.memalign == NULL)) { + (gMallocUse.memalign == NULL) || + (gMallocUse.malloc_usable_size == NULL)) { error_log("%s: some symbols for libc.debug.malloc level %d were not found (see above)", __progname, gMallocDebugLevel); - dlclose(libc_malloc_impl_handle); - libc_malloc_impl_handle = NULL; + dlclose(malloc_impl_handle); } else { __libc_malloc_dispatch = &gMallocUse; + libc_malloc_impl_handle = malloc_impl_handle; } } diff --git a/libc/bionic/malloc_debug_common.h b/libc/bionic/malloc_debug_common.h index 12d0e65e2..a3f9909de 100644 --- a/libc/bionic/malloc_debug_common.h +++ b/libc/bionic/malloc_debug_common.h @@ -45,6 +45,11 @@ #define MAX_SIZE_T (~(size_t)0) +// This must match the alignment used by dlmalloc. +#ifndef MALLOC_ALIGNMENT +#define MALLOC_ALIGNMENT ((size_t)(2 * sizeof(void *))) +#endif + // ============================================================================= // Structures // ============================================================================= @@ -71,12 +76,14 @@ typedef void (*MallocDebugFree)(void*); typedef void* (*MallocDebugCalloc)(size_t, size_t); typedef void* (*MallocDebugRealloc)(void*, size_t); typedef void* (*MallocDebugMemalign)(size_t, size_t); +typedef size_t (*MallocDebugMallocUsableSize)(const void*); struct MallocDebug { MallocDebugMalloc malloc; MallocDebugFree free; MallocDebugCalloc calloc; MallocDebugRealloc realloc; MallocDebugMemalign memalign; + MallocDebugMallocUsableSize malloc_usable_size; }; /* Malloc debugging initialization and finalization routines. diff --git a/libc/bionic/malloc_debug_leak.cpp b/libc/bionic/malloc_debug_leak.cpp index 2db8a1f62..45b45c256 100644 --- a/libc/bionic/malloc_debug_leak.cpp +++ b/libc/bionic/malloc_debug_leak.cpp @@ -67,9 +67,6 @@ extern HashTable gHashTable; // stack trace functions // ============================================================================= -#ifndef MALLOC_ALIGNMENT -#define MALLOC_ALIGNMENT ((size_t)8U) -#endif #define GUARD 0x48151642 #define DEBUG 0 @@ -80,12 +77,16 @@ extern HashTable gHashTable; struct AllocationEntry { HashEntry* entry; uint32_t guard; -}; +} __attribute__((aligned(MALLOC_ALIGNMENT))); -static AllocationEntry* to_header(void* mem) { +static inline AllocationEntry* to_header(void* mem) { return reinterpret_cast(mem) - 1; } +static inline const AllocationEntry* const_to_header(const void* mem) { + return reinterpret_cast(mem) - 1; +} + // ============================================================================= // Hash Table functions // ============================================================================= @@ -229,17 +230,16 @@ extern "C" void fill_free(void* mem) { } extern "C" void* fill_realloc(void* mem, size_t bytes) { - void* buffer = fill_malloc(bytes); - if (mem == NULL) { - return buffer; + size_t oldSize = dlmalloc_usable_size(mem); + void* newMem = dlrealloc(mem, bytes); + if (newMem) { + // If this is larger than before, fill the extra with our pattern. + size_t newSize = dlmalloc_usable_size(newMem); + if (newSize > oldSize) { + memset(reinterpret_cast(reinterpret_cast(newMem)+oldSize), CHK_FILL_FREE, newSize-oldSize); + } } - if (buffer) { - size_t old_size = dlmalloc_usable_size(mem); - size_t size = (bytes < old_size)?(bytes):(old_size); - memcpy(buffer, mem, size); - fill_free(mem); - } - return buffer; + return newMem; } extern "C" void* fill_memalign(size_t alignment, size_t bytes) { @@ -250,11 +250,17 @@ extern "C" void* fill_memalign(size_t alignment, size_t bytes) { return buffer; } +extern "C" size_t fill_malloc_usable_size(const void* mem) { + // Since we didn't allocate extra bytes before or after, we can + // report the normal usable size here. + return dlmalloc_usable_size(mem); +} + // ============================================================================= // malloc leak functions // ============================================================================= -static void* MEMALIGN_GUARD = reinterpret_cast(0xA1A41520); +static uint32_t MEMALIGN_GUARD = 0xA1A41520; extern "C" void* leak_malloc(size_t bytes) { // allocate enough space infront of the allocation to store the pointer for @@ -296,9 +302,10 @@ extern "C" void leak_free(void* mem) { if (header->guard != GUARD) { // could be a memaligned block - if (reinterpret_cast(mem)[-1] == MEMALIGN_GUARD) { - mem = reinterpret_cast(mem)[-2]; - header = to_header(mem); + if (header->guard == MEMALIGN_GUARD) { + // For memaligned blocks, header->entry points to the memory + // allocated through leak_malloc. + header = to_header(header->entry); } } @@ -338,19 +345,26 @@ extern "C" void* leak_realloc(void* oldMem, size_t bytes) { if (oldMem == NULL) { return leak_malloc(bytes); } + void* newMem = NULL; AllocationEntry* header = to_header(oldMem); - if (header && header->guard == GUARD) { - size_t oldSize = header->entry->size & ~SIZE_FLAG_MASK; - newMem = leak_malloc(bytes); - if (newMem != NULL) { - size_t copySize = (oldSize <= bytes) ? oldSize : bytes; - memcpy(newMem, oldMem, copySize); - leak_free(oldMem); - } - } else { - newMem = dlrealloc(oldMem, bytes); + if (header->guard == MEMALIGN_GUARD) { + // Get the real header. + header = to_header(header->entry); + } else if (header->guard != GUARD) { + debug_log("WARNING bad header guard: '0x%x'! and invalid entry: %p\n", + header->guard, header->entry); + return NULL; } + + newMem = leak_malloc(bytes); + if (newMem != NULL) { + size_t oldSize = header->entry->size & ~SIZE_FLAG_MASK; + size_t copySize = (oldSize <= bytes) ? oldSize : bytes; + memcpy(newMem, oldMem, copySize); + } + leak_free(oldMem); + return newMem; } @@ -375,7 +389,7 @@ extern "C" void* leak_memalign(size_t alignment, size_t bytes) { void* base = leak_malloc(size); if (base != NULL) { - intptr_t ptr = reinterpret_cast(base); + uintptr_t ptr = reinterpret_cast(base); if ((ptr % alignment) == 0) { return base; } @@ -383,11 +397,38 @@ extern "C" void* leak_memalign(size_t alignment, size_t bytes) { // align the pointer ptr += ((-ptr) % alignment); - // there is always enough space for the base pointer and the guard - reinterpret_cast(ptr)[-1] = MEMALIGN_GUARD; - reinterpret_cast(ptr)[-2] = base; + // Already allocated enough space for the header. This assumes + // that the malloc alignment is at least 8, otherwise, this is + // not guaranteed to have the space for the header. + AllocationEntry* header = to_header(reinterpret_cast(ptr)); + header->guard = MEMALIGN_GUARD; + header->entry = reinterpret_cast(base); return reinterpret_cast(ptr); } return base; } + +extern "C" size_t leak_malloc_usable_size(const void* mem) { + if (mem != NULL) { + // Check the guard to make sure it is valid. + const AllocationEntry* header = const_to_header((void*)mem); + + if (header->guard == MEMALIGN_GUARD) { + // If this is a memalign'd pointer, then grab the header from + // entry. + header = const_to_header(header->entry); + } else if (header->guard != GUARD) { + debug_log("WARNING bad header guard: '0x%x'! and invalid entry: %p\n", + header->guard, header->entry); + return 0; + } + + size_t ret = dlmalloc_usable_size(header); + if (ret != 0) { + // The usable area starts at 'mem' and stops at 'header+ret'. + return reinterpret_cast(header) + ret - reinterpret_cast(mem); + } + } + return 0; +} diff --git a/libc/bionic/malloc_debug_qemu.cpp b/libc/bionic/malloc_debug_qemu.cpp index 34ddb8751..4c666a90c 100644 --- a/libc/bionic/malloc_debug_qemu.cpp +++ b/libc/bionic/malloc_debug_qemu.cpp @@ -137,7 +137,7 @@ struct MallocDescQuery { * will respond with information about allocated block that contains this * pointer. */ - void* ptr; + const void* ptr; /* Id of the process that initialized libc instance, in which this query * is called. This field is used by the emulator to report errors in @@ -469,7 +469,7 @@ static inline int notify_qemu_free(void* ptr_to_free) { * Return: * Zero on success, or -1 on failure. */ -static inline int query_qemu_malloc_info(void* ptr, MallocDesc* desc, uint32_t routine) { +static inline int query_qemu_malloc_info(const void* ptr, MallocDesc* desc, uint32_t routine) { volatile MallocDescQuery query; query.ptr = ptr; @@ -574,11 +574,12 @@ static void test_access_violation(const MallocDesc* desc) { // API routines // ============================================================================= -void* qemu_instrumented_malloc(size_t bytes); -void qemu_instrumented_free(void* mem); -void* qemu_instrumented_calloc(size_t n_elements, size_t elem_size); -void* qemu_instrumented_realloc(void* mem, size_t bytes); -void* qemu_instrumented_memalign(size_t alignment, size_t bytes); +extern "C" void* qemu_instrumented_malloc(size_t bytes); +extern "C" void qemu_instrumented_free(void* mem); +extern "C" void* qemu_instrumented_calloc(size_t n_elements, size_t elem_size); +extern "C" void* qemu_instrumented_realloc(void* mem, size_t bytes); +extern "C" void* qemu_instrumented_memalign(size_t alignment, size_t bytes); +extern "C" size_t qemu_instrumented_malloc_usable_size(const void* mem); /* Initializes malloc debugging instrumentation for the emulator. * This routine is called from malloc_init_impl routine implemented in @@ -589,7 +590,7 @@ void* qemu_instrumented_memalign(size_t alignment, size_t bytes); * Return: * 0 on success, or -1 on failure. */ -int malloc_debug_initialize() { +extern "C" int malloc_debug_initialize() { /* We will be using emulator's magic page to report memory allocation * activities. In essence, what magic page does, it translates writes to * the memory mapped spaces into writes to an I/O port that emulator @@ -627,7 +628,7 @@ int malloc_debug_initialize() { * Return: * 0 on success, or -1 on failure. */ -int memcheck_initialize(int alignment, const char* memcheck_param) { +extern "C" int memcheck_initialize(int alignment, const char* memcheck_param) { malloc_alignment = alignment; /* Parse -memcheck parameter for the guest tracing flags. */ @@ -673,7 +674,7 @@ int memcheck_initialize(int alignment, const char* memcheck_param) { * bytes (plus prefix, and suffix guards), and report allocation to the * emulator. */ -void* qemu_instrumented_malloc(size_t bytes) { +extern "C" void* qemu_instrumented_malloc(size_t bytes) { MallocDesc desc; /* Initialize block descriptor and allocate memory. Note that dlmalloc @@ -708,7 +709,7 @@ void* qemu_instrumented_malloc(size_t bytes) { * Primary responsibility of this routine is to free requested memory, and * report free block to the emulator. */ -void qemu_instrumented_free(void* mem) { +extern "C" void qemu_instrumented_free(void* mem) { MallocDesc desc; if (mem == NULL) { @@ -751,7 +752,7 @@ void qemu_instrumented_free(void* mem) { /* This routine serves as entry point for 'calloc'. * This routine behaves similarly to qemu_instrumented_malloc. */ -void* qemu_instrumented_calloc(size_t n_elements, size_t elem_size) { +extern "C" void* qemu_instrumented_calloc(size_t n_elements, size_t elem_size) { if (n_elements == 0 || elem_size == 0) { // Just let go zero bytes allocation. qemu_info_log("::: : Zero calloc redir to malloc", @@ -823,7 +824,7 @@ void* qemu_instrumented_calloc(size_t n_elements, size_t elem_size) { * allocation, but overall it doesn't seem to matter, as caller of realloc * should not expect that pointer returned after shrinking will remain the same. */ -void* qemu_instrumented_realloc(void* mem, size_t bytes) { +extern "C" void* qemu_instrumented_realloc(void* mem, size_t bytes) { MallocDesc new_desc; MallocDesc cur_desc; size_t to_copy; @@ -927,7 +928,7 @@ void* qemu_instrumented_realloc(void* mem, size_t bytes) { /* This routine serves as entry point for 'memalign'. * This routine behaves similarly to qemu_instrumented_malloc. */ -void* qemu_instrumented_memalign(size_t alignment, size_t bytes) { +extern "C" void* qemu_instrumented_memalign(size_t alignment, size_t bytes) { MallocDesc desc; if (bytes == 0) { @@ -967,3 +968,27 @@ void* qemu_instrumented_memalign(size_t alignment, size_t bytes) { malloc_pid, getpid(), alignment, bytes); return mallocdesc_user_ptr(&desc); } + +extern "C" size_t qemu_instrumented_malloc_usable_size(const void* mem) { + MallocDesc cur_desc; + + // Query emulator for the reallocating block information. + if (query_qemu_malloc_info(mem, &cur_desc, 2)) { + // Note that this violation should be already caught in the emulator. + error_log(": malloc_usable_size(%p) query_info failed.", + malloc_pid, getpid(), mem); + return 0; + } + + /* Make sure that reallocating pointer value is what we would expect + * for this memory block. Note that this violation should be already caught + * in the emulator.*/ + if (mem != mallocdesc_user_ptr(&cur_desc)) { + log_mdesc(error, &cur_desc, ": malloc_usable_size(%p) is invalid for ", + malloc_pid, getpid(), mem); + return 0; + } + + /* during instrumentation, we can't really report anything more than requested_bytes */ + return cur_desc.requested_bytes; +} diff --git a/tests/Android.mk b/tests/Android.mk index 45cb462ce..875746dcd 100644 --- a/tests/Android.mk +++ b/tests/Android.mk @@ -66,6 +66,7 @@ test_src_files = \ getcwd_test.cpp \ libc_logging_test.cpp \ libgen_test.cpp \ + malloc_test.cpp \ math_test.cpp \ netdb_test.cpp \ pthread_test.cpp \ diff --git a/tests/malloc_test.cpp b/tests/malloc_test.cpp new file mode 100644 index 000000000..259853dca --- /dev/null +++ b/tests/malloc_test.cpp @@ -0,0 +1,235 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +TEST(malloc, malloc_std) { + // Simple malloc test. + void *ptr = malloc(100); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(100U, malloc_usable_size(ptr)); + + free(ptr); +} + +TEST(malloc, calloc_std) { + // Simple calloc test. + size_t alloc_len = 100; + char *ptr = (char *)calloc(1, alloc_len); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(alloc_len, malloc_usable_size(ptr)); + for (size_t i = 0; i < alloc_len; i++) { + ASSERT_EQ(0, ptr[i]); + } + + free(ptr); +} + +TEST(malloc, memalign_multiple) { + // Memalign test where the alignment is any value. + for (size_t i = 0; i <= 12; i++) { + for (size_t alignment = 1 << i; alignment < (1U << (i+1)); alignment++) { + char *ptr = (char*)memalign(alignment, 100); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(100U, malloc_usable_size(ptr)); + ASSERT_EQ(0, (intptr_t)ptr % (1 << i)); + + free(ptr); + } + } +} + +TEST(malloc, memalign_realloc) { + // Memalign and then realloc the pointer a couple of times. + for (size_t alignment = 1; alignment <= 4096; alignment <<= 1) { + char *ptr = (char*)memalign(alignment, 100); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(100U, malloc_usable_size(ptr)); + ASSERT_EQ(0U, (intptr_t)ptr % alignment); + memset(ptr, 0x23, 100); + + ptr = (char*)realloc(ptr, 200); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(200U, malloc_usable_size(ptr)); + ASSERT_TRUE(ptr != NULL); + for (size_t i = 0; i < 100; i++) { + ASSERT_EQ(0x23, ptr[i]); + } + memset(ptr, 0x45, 200); + + ptr = (char*)realloc(ptr, 300); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(300U, malloc_usable_size(ptr)); + for (size_t i = 0; i < 200; i++) { + ASSERT_EQ(0x45, ptr[i]); + } + memset(ptr, 0x67, 300); + + ptr = (char*)realloc(ptr, 250); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(250U, malloc_usable_size(ptr)); + for (size_t i = 0; i < 250; i++) { + ASSERT_EQ(0x67, ptr[i]); + } + + free(ptr); + } +} + +TEST(malloc, malloc_realloc_larger) { + // Realloc to a larger size, malloc is used for the original allocation. + char *ptr = (char *)malloc(100); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(100U, malloc_usable_size(ptr)); + memset(ptr, 67, 100); + + ptr = (char *)realloc(ptr, 200); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(200U, malloc_usable_size(ptr)); + for (size_t i = 0; i < 100; i++) { + ASSERT_EQ(67, ptr[i]); + } + + free(ptr); +} + +TEST(malloc, malloc_realloc_smaller) { + // Realloc to a smaller size, malloc is used for the original allocation. + char *ptr = (char *)malloc(200); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(200U, malloc_usable_size(ptr)); + memset(ptr, 67, 200); + + ptr = (char *)realloc(ptr, 100); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(100U, malloc_usable_size(ptr)); + for (size_t i = 0; i < 100; i++) { + ASSERT_EQ(67, ptr[i]); + } + + free(ptr); +} + +TEST(malloc, malloc_multiple_realloc) { + // Multiple reallocs, malloc is used for the original allocation. + char *ptr = (char *)malloc(200); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(200U, malloc_usable_size(ptr)); + memset(ptr, 0x23, 200); + + ptr = (char *)realloc(ptr, 100); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(100U, malloc_usable_size(ptr)); + for (size_t i = 0; i < 100; i++) { + ASSERT_EQ(0x23, ptr[i]); + } + + ptr = (char*)realloc(ptr, 50); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(50U, malloc_usable_size(ptr)); + for (size_t i = 0; i < 50; i++) { + ASSERT_EQ(0x23, ptr[i]); + } + + ptr = (char*)realloc(ptr, 150); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(150U, malloc_usable_size(ptr)); + for (size_t i = 0; i < 50; i++) { + ASSERT_EQ(0x23, ptr[i]); + } + memset(ptr, 0x23, 150); + + ptr = (char*)realloc(ptr, 425); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(425U, malloc_usable_size(ptr)); + for (size_t i = 0; i < 150; i++) { + ASSERT_EQ(0x23, ptr[i]); + } + + free(ptr); +} +TEST(malloc, calloc_realloc_larger) { + // Realloc to a larger size, calloc is used for the original allocation. + char *ptr = (char *)calloc(1, 100); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(100U, malloc_usable_size(ptr)); + + ptr = (char *)realloc(ptr, 200); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(200U, malloc_usable_size(ptr)); + for (size_t i = 0; i < 100; i++) { + ASSERT_EQ(0, ptr[i]); + } + + free(ptr); +} + +TEST(malloc, calloc_realloc_smaller) { + // Realloc to a smaller size, calloc is used for the original allocation. + char *ptr = (char *)calloc(1, 200); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(200U, malloc_usable_size(ptr)); + + ptr = (char *)realloc(ptr, 100); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(100U, malloc_usable_size(ptr)); + for (size_t i = 0; i < 100; i++) { + ASSERT_EQ(0, ptr[i]); + } + + free(ptr); +} + +TEST(malloc, calloc_multiple_realloc) { + // Multiple reallocs, calloc is used for the original allocation. + char *ptr = (char *)calloc(1, 200); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(200U, malloc_usable_size(ptr)); + + ptr = (char *)realloc(ptr, 100); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(100U, malloc_usable_size(ptr)); + for (size_t i = 0; i < 100; i++) { + ASSERT_EQ(0, ptr[i]); + } + + ptr = (char*)realloc(ptr, 50); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(50U, malloc_usable_size(ptr)); + for (size_t i = 0; i < 50; i++) { + ASSERT_EQ(0, ptr[i]); + } + + ptr = (char*)realloc(ptr, 150); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(150U, malloc_usable_size(ptr)); + for (size_t i = 0; i < 50; i++) { + ASSERT_EQ(0, ptr[i]); + } + memset(ptr, 0, 150); + + ptr = (char*)realloc(ptr, 425); + ASSERT_TRUE(ptr != NULL); + ASSERT_LE(425U, malloc_usable_size(ptr)); + for (size_t i = 0; i < 150; i++) { + ASSERT_EQ(0, ptr[i]); + } + + free(ptr); +} From 87594a32b892d629127c312b78e6cb45d2f678aa Mon Sep 17 00:00:00 2001 From: Chad Brubaker Date: Thu, 6 Jun 2013 15:23:33 -0700 Subject: [PATCH 17/77] Add clearing of resolv uid and pid interface maps Cleanup uid interface map variable names Change-Id: I712dd83276bb23b149af4180515ef33a1bade5ea --- libc/netbsd/resolv/res_cache.c | 74 +++++++++++++++++++++++++--------- libc/private/resolv_iface.h | 10 ++++- 2 files changed, 63 insertions(+), 21 deletions(-) diff --git a/libc/netbsd/resolv/res_cache.c b/libc/netbsd/resolv/res_cache.c index b2d5f1b89..7dd32e019 100644 --- a/libc/netbsd/resolv/res_cache.c +++ b/libc/netbsd/resolv/res_cache.c @@ -1259,8 +1259,8 @@ typedef struct resolv_pidiface_info { struct resolv_pidiface_info* next; } PidIfaceInfo; typedef struct resolv_uidiface_info { - int low; - int high; + int uid_start; + int uid_end; char ifname[IF_NAMESIZE + 1]; struct resolv_uidiface_info* next; } UidIfaceInfo; @@ -1852,9 +1852,9 @@ static void _remove_pidiface_info_locked(int pid); static struct resolv_pidiface_info* _get_pid_iface_info_locked(int pid); /* remove a resolv_pidiface_info structure from _res_uidiface_list */ -static int _remove_uidiface_info_locked(int low, int high); +static int _remove_uidiface_info_locked(int uid_start, int uid_end); /* check if a range [low,high] overlaps with any already existing ranges in the uid=>iface map*/ -static int _resolv_check_uid_range_overlap_locked(int low, int high); +static int _resolv_check_uid_range_overlap_locked(int uid_start, int uid_end); /* get a resolv_uidiface_info structure from _res_uidiface_list with a certain uid */ static struct resolv_uidiface_info* _get_uid_iface_info_locked(int uid); @@ -2433,11 +2433,11 @@ _resolv_get_pids_associated_interface(int pid, char* buff, int buffLen) } static int -_remove_uidiface_info_locked(int low, int high) { - struct resolv_uidiface_info* result = &_res_uidiface_list.next; +_remove_uidiface_info_locked(int uid_start, int uid_end) { + struct resolv_uidiface_info* result = _res_uidiface_list.next; struct resolv_uidiface_info* prev = NULL; - while (result != NULL && result->low != low && result->high != high) { + while (result != NULL && result->uid_start != uid_start && result->uid_end != uid_end) { prev = result; result = result->next; } @@ -2453,8 +2453,8 @@ _remove_uidiface_info_locked(int low, int high) { static struct resolv_uidiface_info* _get_uid_iface_info_locked(int uid) { - struct resolv_uidiface_info* result = &_res_uidiface_list.next; - while (result != NULL && !(result->low <= uid && result->high >= uid)) { + struct resolv_uidiface_info* result = _res_uidiface_list.next; + while (result != NULL && !(result->uid_start <= uid && result->uid_end >= uid)) { result = result->next; } @@ -2462,11 +2462,11 @@ _get_uid_iface_info_locked(int uid) } static int -_resolv_check_uid_range_overlap_locked(int low, int high) +_resolv_check_uid_range_overlap_locked(int uid_start, int uid_end) { - struct resolv_uidiface_info* cur = &_res_uidiface_list.next; + struct resolv_uidiface_info* cur = _res_uidiface_list.next; while (cur != NULL) { - if (cur->low <= high && cur->high >= low) { + if (cur->uid_start <= uid_end && cur->uid_end >= uid_start) { return -1; } cur = cur->next; @@ -2474,20 +2474,56 @@ _resolv_check_uid_range_overlap_locked(int low, int high) return 0; } +void +_resolv_clear_iface_uid_range_mapping() +{ + pthread_once(&_res_cache_once, _res_cache_init); + pthread_mutex_lock(&_res_uidiface_list_lock); + struct resolv_uidiface_info *current = _res_uidiface_list.next; + struct resolv_uidiface_info *next; + while (current != NULL) { + next = current->next; + free(current); + current = next; + } + _res_uidiface_list.next = NULL; + pthread_mutex_unlock(&_res_uidiface_list_lock); +} + +void +_resolv_clear_iface_pid_mapping() +{ + pthread_once(&_res_cache_once, _res_cache_init); + pthread_mutex_lock(&_res_pidiface_list_lock); + struct resolv_pidiface_info *current = _res_pidiface_list.next; + struct resolv_pidiface_info *next; + while (current != NULL) { + next = current->next; + free(current); + current = next; + } + _res_pidiface_list.next = NULL; + pthread_mutex_unlock(&_res_pidiface_list_lock); +} + int -_resolv_set_iface_for_uid_range(const char* ifname, int low, int high) +_resolv_set_iface_for_uid_range(const char* ifname, int uid_start, int uid_end) { int rv = 0; struct resolv_uidiface_info* uidiface_info; // make sure the uid iface list is created pthread_once(&_res_cache_once, _res_cache_init); + if (uid_start > uid_end) { + errno = EINVAL; + return -1; + } pthread_mutex_lock(&_res_uidiface_list_lock); //check that we aren't adding an overlapping range - if (!_resolv_check_uid_range_overlap_locked(low,high)) { + if (!_resolv_check_uid_range_overlap_locked(uid_start, uid_end)) { uidiface_info = calloc(sizeof(*uidiface_info), 1); if (uidiface_info) { - uidiface_info->low = low; - uidiface_info->high = high; + uidiface_info->uid_start = uid_start; + uidiface_info->uid_end = uid_end; int len = sizeof(uidiface_info->ifname); strncpy(uidiface_info->ifname, ifname, len - 1); uidiface_info->ifname[len - 1] = '\0'; @@ -2512,14 +2548,14 @@ _resolv_set_iface_for_uid_range(const char* ifname, int low, int high) } int -_resolv_clear_iface_for_uid_range(int low, int high) +_resolv_clear_iface_for_uid_range(int uid_start, int uid_end) { pthread_once(&_res_cache_once, _res_cache_init); pthread_mutex_lock(&_res_uidiface_list_lock); - int rv = _remove_uidiface_info_locked(low, high); + int rv = _remove_uidiface_info_locked(uid_start, uid_end); - XLOG("_resolv_clear_iface_for_uid_range: low %d high %d\n", low, high); + XLOG("_resolv_clear_iface_for_uid_range: [%d,%d]\n", uid_start, uid_end); pthread_mutex_unlock(&_res_uidiface_list_lock); diff --git a/libc/private/resolv_iface.h b/libc/private/resolv_iface.h index 6be97c1fd..ad427930b 100644 --- a/libc/private/resolv_iface.h +++ b/libc/private/resolv_iface.h @@ -66,6 +66,9 @@ extern void _resolv_set_iface_for_pid(const char* ifname, int pid); /* clear pid from being associated with an interface */ extern void _resolv_clear_iface_for_pid(int pid); +/* clear the entire mapping of pids to interfaces. */ +extern void _resolv_clear_iface_pid_mapping(); + /** Gets the name of the interface to which the pid is attached. * On error, -1 is returned. * If no interface is found, 0 is returned and buff is set to empty ('\0'). @@ -78,11 +81,14 @@ extern int _resolv_get_pids_associated_interface(int pid, char* buff, int buffLe /** set a uid range to use the name servers of the specified interface * If [low,high] overlaps with an already existing rule -1 is returned */ -extern int _resolv_set_iface_for_uid_range(const char* ifname, int low, int high); +extern int _resolv_set_iface_for_uid_range(const char* ifname, int uid_start, int uid_end); /* clear a uid range from being associated with an interface * If the range given is not mapped -1 is returned. */ -extern int _resolv_clear_iface_for_uid_range(int low, int high); +extern int _resolv_clear_iface_for_uid_range(int uid_start, int uid_end); + +/* clear the entire mapping of uid ranges to interfaces. */ +extern void _resolv_clear_iface_uid_range_mapping(); /** Gets the name of the interface to which the uid is attached. * On error, -1 is returned. From 638503b5157cc19ad58468cc01ad48cfb38e6435 Mon Sep 17 00:00:00 2001 From: Chad Brubaker Date: Fri, 14 Jun 2013 15:25:42 -0700 Subject: [PATCH 18/77] Fix remove_uidiface_info failing on first entry Change-Id: Ic23506581ff835a6b679e1593eab550a84548056 --- libc/netbsd/resolv/res_cache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/netbsd/resolv/res_cache.c b/libc/netbsd/resolv/res_cache.c index 7dd32e019..8e1bd14ce 100644 --- a/libc/netbsd/resolv/res_cache.c +++ b/libc/netbsd/resolv/res_cache.c @@ -2435,7 +2435,7 @@ _resolv_get_pids_associated_interface(int pid, char* buff, int buffLen) static int _remove_uidiface_info_locked(int uid_start, int uid_end) { struct resolv_uidiface_info* result = _res_uidiface_list.next; - struct resolv_uidiface_info* prev = NULL; + struct resolv_uidiface_info* prev = &_res_uidiface_list; while (result != NULL && result->uid_start != uid_start && result->uid_end != uid_end) { prev = result; From dc1038b7900acb664e99643d2974e1a0f4703781 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Wed, 23 Jan 2013 23:07:06 -0800 Subject: [PATCH 19/77] bionic: move system property writing from init to bionic Move the implementation of writing to the system property area from init to bionic, next to the reader implementation. This will allow full property testing to be added to bionic tests. Add new accessor and waiting functions to hide the implementation from watchprops and various bionic users. Also hide some of the implementation details of the property area from init by moving them into _system_properties.h, and other details from everybody by moving them into system_properties.h. Change-Id: I9026e604109e30546b2849b60cab2e7e5ff00ba5 --- libc/bionic/system_properties.c | 101 ++++++++++++++++++++++++++ libc/include/sys/_system_properties.h | 66 ++++++++++++----- libc/netbsd/resolv/res_state.c | 6 +- 3 files changed, 153 insertions(+), 20 deletions(-) diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index 0587430a5..d96950709 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -26,6 +26,7 @@ * SUCH DAMAGE. */ #include +#include #include #include #include @@ -49,6 +50,25 @@ #include +struct prop_area { + unsigned volatile count; + unsigned volatile serial; + unsigned magic; + unsigned version; + unsigned reserved[4]; + unsigned toc[1]; +}; + +typedef struct prop_area prop_area; + +struct prop_info { + char name[PROP_NAME_MAX]; + unsigned volatile serial; + char value[PROP_VALUE_MAX]; +}; + +typedef struct prop_info prop_info; + static const char property_service_socket[] = "/dev/socket/" PROP_SERVICE_NAME; static unsigned dummy_props = 0; @@ -66,6 +86,17 @@ static int get_fd_from_env(void) return atoi(env); } +void __system_property_area_init(void *data) +{ + prop_area *pa = data; + memset(pa, 0, PA_SIZE); + pa->magic = PROP_AREA_MAGIC; + pa->version = PROP_AREA_VERSION; + + /* plug into the lib property services */ + __system_property_area__ = pa; +} + int __system_properties_init(void) { bool fromFile = true; @@ -147,6 +178,11 @@ const prop_info *__system_property_find(const char *name) unsigned len = strlen(name); prop_info *pi; + if (len >= PROP_NAME_MAX) + return 0; + if (len < 1) + return 0; + while(count--) { unsigned entry = *toc++; if(TOC_NAME_LEN(entry) != len) continue; @@ -294,3 +330,68 @@ int __system_property_wait(const prop_info *pi) } return 0; } + +int __system_property_update(prop_info *pi, const char *value, unsigned int len) +{ + prop_area *pa = __system_property_area__; + + if (len >= PROP_VALUE_MAX) + return -1; + + pi->serial = pi->serial | 1; + memcpy(pi->value, value, len + 1); + pi->serial = (len << 24) | ((pi->serial + 1) & 0xffffff); + __futex_wake(&pi->serial, INT32_MAX); + + pa->serial++; + __futex_wake(&pa->serial, INT32_MAX); + + return 0; +} + +int __system_property_add(const char *name, unsigned int namelen, + const char *value, unsigned int valuelen) +{ + prop_area *pa = __system_property_area__; + prop_info *pa_info_array = (void*) (((char*) pa) + PA_INFO_START); + prop_info *pi; + + if (pa->count == PA_COUNT_MAX) + return -1; + if (namelen >= PROP_NAME_MAX) + return -1; + if (valuelen >= PROP_VALUE_MAX) + return -1; + if (namelen < 1) + return -1; + + pi = pa_info_array + pa->count; + pi->serial = (valuelen << 24); + memcpy(pi->name, name, namelen + 1); + memcpy(pi->value, value, valuelen + 1); + + pa->toc[pa->count] = + (namelen << 24) | (((unsigned) pi) - ((unsigned) pa)); + + pa->count++; + pa->serial++; + __futex_wake(&pa->serial, INT32_MAX); + + return 0; +} + +unsigned int __system_property_serial(const prop_info *pi) +{ + return pi->serial; +} + +unsigned int __system_property_wait_any(unsigned int serial) +{ + prop_area *pa = __system_property_area__; + + do { + __futex_wait(&pa->serial, serial, 0); + } while(pa->serial == serial); + + return pa->serial; +} diff --git a/libc/include/sys/_system_properties.h b/libc/include/sys/_system_properties.h index 5d2043d24..c5bc2235b 100644 --- a/libc/include/sys/_system_properties.h +++ b/libc/include/sys/_system_properties.h @@ -34,7 +34,6 @@ #else #include -typedef struct prop_area prop_area; typedef struct prop_msg prop_msg; #define PROP_AREA_MAGIC 0x504f5250 @@ -43,29 +42,20 @@ typedef struct prop_msg prop_msg; #define PROP_SERVICE_NAME "property_service" #define PROP_FILENAME "/dev/__properties__" -/* #define PROP_MAX_ENTRIES 247 */ -/* 247 -> 32620 bytes (<32768) */ +/* (8 header words + 247 toc words) = 1020 bytes */ +/* 1024 bytes header and toc + 247 prop_infos @ 128 bytes = 32640 bytes */ + +#define PA_COUNT_MAX 247 +#define PA_INFO_START 1024 +#define PA_SIZE 32768 #define TOC_NAME_LEN(toc) ((toc) >> 24) #define TOC_TO_INFO(area, toc) ((prop_info*) (((char*) area) + ((toc) & 0xFFFFFF))) -struct prop_area { - unsigned volatile count; - unsigned volatile serial; - unsigned magic; - unsigned version; - unsigned reserved[4]; - unsigned toc[1]; -}; - #define SERIAL_VALUE_LEN(serial) ((serial) >> 24) #define SERIAL_DIRTY(serial) ((serial) & 1) -struct prop_info { - char name[PROP_NAME_MAX]; - unsigned volatile serial; - char value[PROP_VALUE_MAX]; -}; +__BEGIN_DECLS struct prop_msg { @@ -106,5 +96,47 @@ struct prop_msg #define PROP_PATH_LOCAL_OVERRIDE "/data/local.prop" #define PROP_PATH_FACTORY "/factory/factory.prop" +/* +** Initialize the area to be used to store properties. Can +** only be done by a single process that has write access to +** the property area. +*/ +void __system_property_area_init(void *data); + +/* Add a new system property. Can only be done by a single +** process that has write access to the property area, and +** that process must handle sequencing to ensure the property +** does not already exist and that only one property is added +** or updated at a time. +** +** Returns 0 on success, -1 if the property area is full. +*/ +int __system_property_add(const char *name, unsigned int namelen, + const char *value, unsigned int valuelen); + +/* Update the value of a system property returned by +** __system_property_find. Can only be done by a single process +** that has write access to the property area, and that process +** must handle sequencing to ensure that only one property is +** updated at a time. +** +** Returns 0 on success, -1 if the parameters are incorrect. +*/ +int __system_property_update(prop_info *pi, const char *value, unsigned int len); + +/* Read the serial number of a system property returned by +** __system_property_find. +** +** Returns the serial number on success, -1 on error. +*/ +unsigned int __system_property_serial(const prop_info *pi); + +/* Wait for any system property to be updated. Caller must pass +** in 0 the first time, and the previous return value on each +** successive call. */ +unsigned int __system_property_wait_any(unsigned int serial); + +__END_DECLS + #endif #endif diff --git a/libc/netbsd/resolv/res_state.c b/libc/netbsd/resolv/res_state.c index 32ffdcaf8..de82e1a54 100644 --- a/libc/netbsd/resolv/res_state.c +++ b/libc/netbsd/resolv/res_state.c @@ -71,7 +71,7 @@ _res_thread_alloc(void) rt->_serial = 0; rt->_pi = (struct prop_info*) __system_property_find("net.change"); if (rt->_pi) { - rt->_serial = rt->_pi->serial; + rt->_serial = __system_property_serial(rt->_pi); } memset(rt->_rstatic, 0, sizeof rt->_rstatic); } @@ -135,14 +135,14 @@ _res_thread_get(void) return rt; } } - if (rt->_serial == rt->_pi->serial) { + if (rt->_serial == __system_property_serial(rt->_pi)) { /* Nothing changed, so return the current state */ D("%s: tid=%d rt=%p nothing changed, returning", __FUNCTION__, gettid(), rt); return rt; } /* Update the recorded serial number, and go reset the state */ - rt->_serial = rt->_pi->serial; + rt->_serial = __system_property_serial(rt->_pi); goto RESET_STATE; } From 37d9f75dde881a0ba1c1b3253b1be19d4096963d Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Mon, 28 Jan 2013 17:19:43 -0800 Subject: [PATCH 20/77] bionic: add tests for properties Change-Id: I09b212966f1c9624631653ef2c7a71de78bbbec1 --- tests/Android.mk | 2 + tests/property_benchmark.cpp | 116 ++++++++++++++++ tests/system_properties_test.cpp | 229 +++++++++++++++++++++++++++++++ 3 files changed, 347 insertions(+) create mode 100644 tests/property_benchmark.cpp create mode 100644 tests/system_properties_test.cpp diff --git a/tests/Android.mk b/tests/Android.mk index 875746dcd..c65d20c34 100644 --- a/tests/Android.mk +++ b/tests/Android.mk @@ -31,6 +31,7 @@ benchmark_c_flags = \ benchmark_src_files = \ benchmark_main.cpp \ math_benchmark.cpp \ + property_benchmark.cpp \ string_benchmark.cpp \ time_benchmark.cpp \ @@ -78,6 +79,7 @@ test_src_files = \ string_test.cpp \ strings_test.cpp \ stubs_test.cpp \ + system_properties_test.cpp \ time_test.cpp \ unistd_test.cpp \ diff --git a/tests/property_benchmark.cpp b/tests/property_benchmark.cpp new file mode 100644 index 000000000..dbd11adcd --- /dev/null +++ b/tests/property_benchmark.cpp @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "benchmark.h" + +#define _REALLY_INCLUDE_SYS__SYSTEM_PROPERTIES_H_ +#include + +#include + +extern void *__system_property_area__; + +#define TEST_NUM_PROPS \ + Arg(1)->Arg(4)->Arg(16)->Arg(64)->Arg(128)->Arg(247) + +struct LocalPropertyTestState { + LocalPropertyTestState(int nprops) : nprops(nprops) { + static const char prop_name_chars[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-_"; + old_pa = __system_property_area__; + pa = malloc(PA_SIZE); + __system_property_area_init(pa); + + names = new char* [nprops]; + name_lens = new int[nprops]; + values = new char* [nprops]; + value_lens = new int[nprops]; + + srandom(nprops); + + for (int i = 0; i < nprops; i++) { + name_lens[i] = random() % PROP_NAME_MAX; + names[i] = new char[PROP_NAME_MAX + 1]; + for (int j = 0; j < name_lens[i]; j++) { + names[i][j] = prop_name_chars[random() % (sizeof(prop_name_chars) - 1)]; + } + names[i][name_lens[i]] = 0; + value_lens[i] = random() % PROP_VALUE_MAX; + values[i] = new char[PROP_VALUE_MAX]; + for (int j = 0; j < value_lens[i]; j++) { + values[i][j] = prop_name_chars[random() % (sizeof(prop_name_chars) - 1)]; + } + __system_property_add(names[i], name_lens[i], values[i], value_lens[i]); + } + } + + ~LocalPropertyTestState() { + __system_property_area__ = old_pa; + for (int i = 0; i < nprops; i++) { + delete names[i]; + delete values[i]; + } + delete names; + delete name_lens; + delete values; + delete value_lens; + free(pa); + } +public: + const int nprops; + char **names; + int *name_lens; + char **values; + int *value_lens; + +private: + void *pa; + void *old_pa; +}; + +static void BM_property_get(int iters, int nprops) +{ + StopBenchmarkTiming(); + + LocalPropertyTestState pa(nprops); + char value[PROP_VALUE_MAX]; + + StartBenchmarkTiming(); + + for (int i = 0; i < iters; i++) { + for (int j = 0; j < nprops; j++) { + __system_property_get(pa.names[j], value); + } + } + StopBenchmarkTiming(); +} +BENCHMARK(BM_property_get)->TEST_NUM_PROPS; + +static void BM_property_find(int iters, int nprops) +{ + StopBenchmarkTiming(); + + LocalPropertyTestState pa(nprops); + + StartBenchmarkTiming(); + + for (int i = 0; i < iters; i++) { + for (int j = 0; j < nprops; j++) { + __system_property_find(pa.names[j]); + } + } + StopBenchmarkTiming(); +} +BENCHMARK(BM_property_find)->TEST_NUM_PROPS; diff --git a/tests/system_properties_test.cpp b/tests/system_properties_test.cpp new file mode 100644 index 000000000..60188f41d --- /dev/null +++ b/tests/system_properties_test.cpp @@ -0,0 +1,229 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#if __BIONIC__ + +#define _REALLY_INCLUDE_SYS__SYSTEM_PROPERTIES_H_ +#include + +extern void *__system_property_area__; + +struct LocalPropertyTestState { + LocalPropertyTestState() { + old_pa = __system_property_area__; + pa = malloc(PA_SIZE); + __system_property_area_init(pa); + } + + ~LocalPropertyTestState() { + __system_property_area__ = old_pa; + free(pa); + } +private: + void *pa; + void *old_pa; +}; + +TEST(properties, add) { + LocalPropertyTestState pa; + + char propvalue[PROP_VALUE_MAX]; + + ASSERT_EQ(0, __system_property_add("property", 8, "value1", 6)); + ASSERT_EQ(0, __system_property_add("other_property", 14, "value2", 6)); + ASSERT_EQ(0, __system_property_add("property_other", 14, "value3", 6)); + + ASSERT_EQ(6, __system_property_get("property", propvalue)); + ASSERT_STREQ(propvalue, "value1"); + + ASSERT_EQ(6, __system_property_get("other_property", propvalue)); + ASSERT_STREQ(propvalue, "value2"); + + ASSERT_EQ(6, __system_property_get("property_other", propvalue)); + ASSERT_STREQ(propvalue, "value3"); +} + +TEST(properties, update) { + LocalPropertyTestState pa; + + char propvalue[PROP_VALUE_MAX]; + prop_info *pi; + + ASSERT_EQ(0, __system_property_add("property", 8, "oldvalue1", 9)); + ASSERT_EQ(0, __system_property_add("other_property", 14, "value2", 6)); + ASSERT_EQ(0, __system_property_add("property_other", 14, "value3", 6)); + + pi = (prop_info *)__system_property_find("property"); + ASSERT_NE((prop_info *)NULL, pi); + __system_property_update(pi, "value4", 6); + + pi = (prop_info *)__system_property_find("other_property"); + ASSERT_NE((prop_info *)NULL, pi); + __system_property_update(pi, "newvalue5", 9); + + pi = (prop_info *)__system_property_find("property_other"); + ASSERT_NE((prop_info *)NULL, pi); + __system_property_update(pi, "value6", 6); + + ASSERT_EQ(6, __system_property_get("property", propvalue)); + ASSERT_STREQ(propvalue, "value4"); + + ASSERT_EQ(9, __system_property_get("other_property", propvalue)); + ASSERT_STREQ(propvalue, "newvalue5"); + + ASSERT_EQ(6, __system_property_get("property_other", propvalue)); + ASSERT_STREQ(propvalue, "value6"); +} + +// 247 = max # of properties supported by current implementation +// (this should never go down) +TEST(properties, fill_247) { + LocalPropertyTestState pa; + char prop_name[PROP_NAME_MAX]; + char prop_value[PROP_VALUE_MAX]; + char prop_value_ret[PROP_VALUE_MAX]; + int ret; + + for (int i = 0; i < 247; i++) { + ret = snprintf(prop_name, PROP_NAME_MAX - 1, "property_%d", i); + memset(prop_name + ret, 'a', PROP_NAME_MAX - 1 - ret); + ret = snprintf(prop_value, PROP_VALUE_MAX - 1, "value_%d", i); + memset(prop_value + ret, 'b', PROP_VALUE_MAX - 1 - ret); + prop_name[PROP_NAME_MAX - 1] = 0; + prop_value[PROP_VALUE_MAX - 1] = 0; + + ASSERT_EQ(0, __system_property_add(prop_name, PROP_NAME_MAX - 1, prop_value, PROP_VALUE_MAX - 1)); + } + + for (int i = 0; i < 247; i++) { + ret = snprintf(prop_name, PROP_NAME_MAX - 1, "property_%d", i); + memset(prop_name + ret, 'a', PROP_NAME_MAX - 1 - ret); + ret = snprintf(prop_value, PROP_VALUE_MAX - 1, "value_%d", i); + memset(prop_value + ret, 'b', PROP_VALUE_MAX - 1 - ret); + prop_name[PROP_NAME_MAX - 1] = 0; + prop_value[PROP_VALUE_MAX - 1] = 0; + memset(prop_value_ret, '\0', PROP_VALUE_MAX); + + ASSERT_EQ(PROP_VALUE_MAX - 1, __system_property_get(prop_name, prop_value_ret)); + ASSERT_EQ(0, memcmp(prop_value, prop_value_ret, PROP_VALUE_MAX)); + } +} + +TEST(properties, find_nth) { + LocalPropertyTestState pa; + + ASSERT_EQ(0, __system_property_add("property", 8, "value1", 6)); + ASSERT_EQ(0, __system_property_add("other_property", 14, "value2", 6)); + ASSERT_EQ(0, __system_property_add("property_other", 14, "value3", 6)); + + ASSERT_NE((const prop_info *)NULL, __system_property_find_nth(0)); + ASSERT_NE((const prop_info *)NULL, __system_property_find_nth(1)); + ASSERT_NE((const prop_info *)NULL, __system_property_find_nth(2)); + + ASSERT_EQ((const prop_info *)NULL, __system_property_find_nth(3)); + ASSERT_EQ((const prop_info *)NULL, __system_property_find_nth(4)); + ASSERT_EQ((const prop_info *)NULL, __system_property_find_nth(5)); + ASSERT_EQ((const prop_info *)NULL, __system_property_find_nth(100)); + ASSERT_EQ((const prop_info *)NULL, __system_property_find_nth(200)); + ASSERT_EQ((const prop_info *)NULL, __system_property_find_nth(247)); +} + +TEST(properties, errors) { + LocalPropertyTestState pa; + char prop_value[PROP_NAME_MAX]; + + ASSERT_EQ(0, __system_property_add("property", 8, "value1", 6)); + ASSERT_EQ(0, __system_property_add("other_property", 14, "value2", 6)); + ASSERT_EQ(0, __system_property_add("property_other", 14, "value3", 6)); + + ASSERT_EQ(0, __system_property_find("property1")); + ASSERT_EQ(0, __system_property_get("property1", prop_value)); + + ASSERT_EQ(-1, __system_property_add("name", PROP_NAME_MAX, "value", 5)); + ASSERT_EQ(-1, __system_property_add("name", 4, "value", PROP_VALUE_MAX)); + ASSERT_EQ(-1, __system_property_update(NULL, "value", PROP_VALUE_MAX)); +} + +TEST(properties, serial) { + LocalPropertyTestState pa; + const prop_info *pi; + unsigned int serial; + + ASSERT_EQ(0, __system_property_add("property", 8, "value1", 6)); + ASSERT_NE((const prop_info *)NULL, pi = __system_property_find("property")); + serial = __system_property_serial(pi); + ASSERT_EQ(0, __system_property_update((prop_info *)pi, "value2", 6)); + ASSERT_NE(serial, __system_property_serial(pi)); +} + +static void *PropertyWaitHelperFn(void *arg) +{ + int *flag = (int *)arg; + prop_info *pi; + pi = (prop_info *)__system_property_find("property"); + usleep(100000); + + *flag = 1; + __system_property_update(pi, "value3", 6); + + return NULL; +} + +TEST(properties, wait) { + LocalPropertyTestState pa; + unsigned int serial; + prop_info *pi; + pthread_t t; + int flag = 0; + + ASSERT_EQ(0, __system_property_add("property", 8, "value1", 6)); + serial = __system_property_wait_any(0); + pi = (prop_info *)__system_property_find("property"); + ASSERT_NE((prop_info *)NULL, pi); + __system_property_update(pi, "value2", 6); + serial = __system_property_wait_any(serial); + + ASSERT_EQ(0, pthread_create(&t, NULL, PropertyWaitHelperFn, &flag)); + ASSERT_EQ(flag, 0); + serial = __system_property_wait_any(serial); + ASSERT_EQ(flag, 1); + + void* result; + ASSERT_EQ(0, pthread_join(t, &result)); +} + +class KilledByFault { + public: + explicit KilledByFault() {}; + bool operator()(int exit_status) const; +}; + +bool KilledByFault::operator()(int exit_status) const { + return WIFSIGNALED(exit_status) && + (WTERMSIG(exit_status) == SIGSEGV || + WTERMSIG(exit_status) == SIGBUS || + WTERMSIG(exit_status) == SIGABRT); +} + +TEST(properties_DeathTest, read_only) { + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + ASSERT_EXIT(__system_property_add("property", 8, "value", 5), + KilledByFault(), ""); +} +#endif From 577418403d68e663fb33c7b0c8a90d862d9c00cf Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Tue, 12 Feb 2013 16:39:31 -0800 Subject: [PATCH 21/77] bionic: add __system_property_foreach find_nth() will be inefficient on a trie. Since find_nth() is only used internally and only for enumerating properties, we can add a foreach() function to do this directly. Change-Id: I66bde9926c193073d74b244cce9fffd52108fff8 Signed-off-by: Greg Hackmann --- libc/bionic/system_properties.c | 16 ++++++++++++++++ libc/include/sys/system_properties.h | 19 ++++++++++++++++--- tests/system_properties_test.cpp | 19 +++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index d96950709..5197ef3ff 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -159,6 +159,22 @@ cleanup: return result; } +int __system_property_foreach( + void (*propfn)(const prop_info *pi, void *cookie), + void *cookie) +{ + prop_area *pa = __system_property_area__; + unsigned i; + + for (i = 0; i < pa->count; i++) { + unsigned entry = pa->toc[i]; + prop_info *pi = TOC_TO_INFO(pa, entry); + propfn(pi, cookie); + } + + return 0; +} + const prop_info *__system_property_find_nth(unsigned n) { prop_area *pa = __system_property_area__; diff --git a/libc/include/sys/system_properties.h b/libc/include/sys/system_properties.h index 85915b2de..01c3db332 100644 --- a/libc/include/sys/system_properties.h +++ b/libc/include/sys/system_properties.h @@ -76,13 +76,26 @@ int __system_property_read(const prop_info *pi, char *name, char *value); ** there is no nth property. Use __system_property_read() to ** read the value of this property. ** -** This method is for inspecting and debugging the property +** Please do not call this method. It only exists to provide +** backwards compatibility to NDK apps. Its implementation +** is inefficient and order of results may change from call +** to call. +*/ +const prop_info *__system_property_find_nth(unsigned n); + +/* Pass a prop_info for each system property to the provided +** callback. Use __system_property_read() to read the value +** of this property. +** +** This method is for inspecting and debugging the property ** system. Please use __system_property_find() instead. ** ** Order of results may change from call to call. This is ** not a bug. -*/ -const prop_info *__system_property_find_nth(unsigned n); +*/ +int __system_property_foreach( + void (*propfn)(const prop_info *pi, void *cookie), + void *cookie); __END_DECLS diff --git a/tests/system_properties_test.cpp b/tests/system_properties_test.cpp index 60188f41d..70ff1d662 100644 --- a/tests/system_properties_test.cpp +++ b/tests/system_properties_test.cpp @@ -125,6 +125,25 @@ TEST(properties, fill_247) { } } +static void foreach_test_callback(const prop_info *pi, void* cookie) { + size_t *count = static_cast(cookie); + + ASSERT_NE((prop_info *)NULL, pi); + (*count)++; +} + +TEST(properties, foreach) { + LocalPropertyTestState pa; + size_t count = 0; + + ASSERT_EQ(0, __system_property_add("property", 8, "value1", 6)); + ASSERT_EQ(0, __system_property_add("other_property", 14, "value2", 6)); + ASSERT_EQ(0, __system_property_add("property_other", 14, "value3", 6)); + + ASSERT_EQ(0, __system_property_foreach(foreach_test_callback, &count)); + ASSERT_EQ(3U, count); +} + TEST(properties, find_nth) { LocalPropertyTestState pa; From d32969701be070c0161c2643ee3c3df16066bbb8 Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Wed, 13 Feb 2013 14:41:48 -0800 Subject: [PATCH 22/77] bionic: make property area expandable The property area is initially one 4K region, automatically expanding as needed up to 64 regions. To avoid duplicating code, __system_property_area_init() now allocates and initializes the first region (previously it was allocated in init's init_property_area() and initialized in bionic). For testing purposes, __system_property_set_filename() may be used to override the file used to map in regions. Change-Id: Ibe00ef52464bfa590953c4699a6d98383b0142b1 Signed-off-by: Greg Hackmann --- libc/bionic/system_properties.c | 187 ++++++++++++++++++++------ libc/include/sys/_system_properties.h | 19 ++- tests/property_benchmark.cpp | 56 ++++++-- tests/system_properties_test.cpp | 76 ++++++++--- 4 files changed, 264 insertions(+), 74 deletions(-) diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index 5197ef3ff..800c8b84f 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -55,7 +55,6 @@ struct prop_area { unsigned volatile serial; unsigned magic; unsigned version; - unsigned reserved[4]; unsigned toc[1]; }; @@ -70,10 +69,9 @@ struct prop_info { typedef struct prop_info prop_info; static const char property_service_socket[] = "/dev/socket/" PROP_SERVICE_NAME; +static char property_filename[PATH_MAX] = PROP_FILENAME; -static unsigned dummy_props = 0; - -prop_area *__system_property_area__ = (void*) &dummy_props; +prop_area *__system_property_regions__[PA_REGION_COUNT] = { NULL, }; static int get_fd_from_env(void) { @@ -86,27 +84,79 @@ static int get_fd_from_env(void) return atoi(env); } -void __system_property_area_init(void *data) +static int map_prop_region_rw(size_t region) { - prop_area *pa = data; + prop_area *pa; + int fd; + size_t offset = region * PA_SIZE; + + if (__system_property_regions__[region]) { + return 0; + } + + /* dev is a tmpfs that we can use to carve a shared workspace + * out of, so let's do that... + */ + fd = open(property_filename, O_RDWR | O_CREAT | O_NOFOLLOW, 0644); + if (fd < 0) { + if (errno == EACCES) { + /* for consistency with the case where the process has already + * mapped the page in and segfaults when trying to write to it + */ + abort(); + } + return -1; + } + + if (ftruncate(fd, offset + PA_SIZE) < 0) + goto out; + + pa = mmap(NULL, PA_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, offset); + if(pa == MAP_FAILED) + goto out; + memset(pa, 0, PA_SIZE); pa->magic = PROP_AREA_MAGIC; pa->version = PROP_AREA_VERSION; /* plug into the lib property services */ - __system_property_area__ = pa; + __system_property_regions__[region] = pa; + + close(fd); + return 0; + +out: + close(fd); + return -1; } -int __system_properties_init(void) +int __system_property_set_filename(const char *filename) +{ + size_t len = strlen(filename); + if (len >= sizeof(property_filename)) + return -1; + + strcpy(property_filename, filename); + return 0; +} + +int __system_property_area_init() +{ + return map_prop_region_rw(0); +} + +static int map_prop_region(size_t region) { bool fromFile = true; + bool swapped; + size_t offset = region * PA_SIZE; int result = -1; - if(__system_property_area__ != ((void*) &dummy_props)) { + if(__system_property_regions__[region]) { return 0; } - int fd = open(PROP_FILENAME, O_RDONLY | O_NOFOLLOW); + int fd = open(property_filename, O_RDONLY | O_NOFOLLOW); if ((fd < 0) && (errno == ENOENT)) { /* @@ -133,23 +183,33 @@ int __system_properties_init(void) if ((fd_stat.st_uid != 0) || (fd_stat.st_gid != 0) - || ((fd_stat.st_mode & (S_IWGRP | S_IWOTH)) != 0)) { + || ((fd_stat.st_mode & (S_IWGRP | S_IWOTH)) != 0) + || (fd_stat.st_size < offset + PA_SIZE) ) { goto cleanup; } - prop_area *pa = mmap(NULL, fd_stat.st_size, PROT_READ, MAP_SHARED, fd, 0); + prop_area *pa = mmap(NULL, PA_SIZE, PROT_READ, MAP_SHARED, fd, offset); if (pa == MAP_FAILED) { goto cleanup; } if((pa->magic != PROP_AREA_MAGIC) || (pa->version != PROP_AREA_VERSION)) { - munmap(pa, fd_stat.st_size); + munmap(pa, PA_SIZE); goto cleanup; } - __system_property_area__ = pa; result = 0; + swapped = __sync_bool_compare_and_swap(&__system_property_regions__[region], + NULL, pa); + if (!swapped) { + /** + * In the event of a race either mapping is equally good, so + * the thread that lost can just throw its mapping away and proceed as + * normal. + */ + munmap(pa, PA_SIZE); + } cleanup: if (fromFile) { @@ -159,17 +219,31 @@ cleanup: return result; } +int __system_properties_init() +{ + return map_prop_region(0); +} + int __system_property_foreach( void (*propfn)(const prop_info *pi, void *cookie), void *cookie) { - prop_area *pa = __system_property_area__; - unsigned i; + size_t region; - for (i = 0; i < pa->count; i++) { - unsigned entry = pa->toc[i]; - prop_info *pi = TOC_TO_INFO(pa, entry); - propfn(pi, cookie); + for (region = 0; region < PA_REGION_COUNT; region++) { + prop_area *pa; + unsigned i; + + int err = map_prop_region(region); + if (err < 0) + break; + pa = __system_property_regions__[region]; + + for (i = 0; i < pa->count; i++) { + unsigned entry = pa->toc[i]; + prop_info *pi = TOC_TO_INFO(pa, entry); + propfn(pi, cookie); + } } return 0; @@ -177,9 +251,15 @@ int __system_property_foreach( const prop_info *__system_property_find_nth(unsigned n) { - prop_area *pa = __system_property_area__; + size_t region = n / PA_COUNT_MAX; + prop_area *pa; - if(n >= pa->count) { + int err = map_prop_region(region); + if (err < 0) + return NULL; + pa = __system_property_regions__[region]; + + if((n % PA_COUNT_MAX) >= pa->count) { return 0; } else { return TOC_TO_INFO(pa, pa->toc[n]); @@ -188,25 +268,36 @@ const prop_info *__system_property_find_nth(unsigned n) const prop_info *__system_property_find(const char *name) { - prop_area *pa = __system_property_area__; - unsigned count = pa->count; - unsigned *toc = pa->toc; unsigned len = strlen(name); - prop_info *pi; + size_t region; if (len >= PROP_NAME_MAX) return 0; if (len < 1) return 0; - while(count--) { - unsigned entry = *toc++; - if(TOC_NAME_LEN(entry) != len) continue; + for (region = 0; region < PA_REGION_COUNT; region++) { + prop_area *pa; + unsigned count; + unsigned *toc; + prop_info *pi; - pi = TOC_TO_INFO(pa, entry); - if(memcmp(name, pi->name, len)) continue; + int err = map_prop_region(region); + if (err < 0) + return 0; + pa = __system_property_regions__[region]; + count = pa->count; + toc = pa->toc; - return pi; + while(count--) { + unsigned entry = *toc++; + if(TOC_NAME_LEN(entry) != len) continue; + + pi = TOC_TO_INFO(pa, entry); + if(memcmp(name, pi->name, len)) continue; + + return pi; + } } return 0; @@ -333,7 +424,7 @@ int __system_property_wait(const prop_info *pi) { unsigned n; if(pi == 0) { - prop_area *pa = __system_property_area__; + prop_area *pa = __system_property_regions__[0]; n = pa->serial; do { __futex_wait(&pa->serial, n, 0); @@ -349,7 +440,7 @@ int __system_property_wait(const prop_info *pi) int __system_property_update(prop_info *pi, const char *value, unsigned int len) { - prop_area *pa = __system_property_area__; + prop_area *pa = __system_property_regions__[0]; if (len >= PROP_VALUE_MAX) return -1; @@ -368,12 +459,11 @@ int __system_property_update(prop_info *pi, const char *value, unsigned int len) int __system_property_add(const char *name, unsigned int namelen, const char *value, unsigned int valuelen) { - prop_area *pa = __system_property_area__; - prop_info *pa_info_array = (void*) (((char*) pa) + PA_INFO_START); + prop_area *pa; + prop_info *pa_info_array; prop_info *pi; + size_t region; - if (pa->count == PA_COUNT_MAX) - return -1; if (namelen >= PROP_NAME_MAX) return -1; if (valuelen >= PROP_VALUE_MAX) @@ -381,13 +471,28 @@ int __system_property_add(const char *name, unsigned int namelen, if (namelen < 1) return -1; + for (region = 0; region < PA_REGION_COUNT; region++) + { + int err = map_prop_region_rw(region); + if (err < 0) + return -1; + + pa = __system_property_regions__[region]; + + if (pa->count < PA_COUNT_MAX) + break; + } + + if (region == PA_REGION_COUNT) + return -1; + + pa_info_array = (void*) (((char*) pa) + PA_INFO_START); pi = pa_info_array + pa->count; pi->serial = (valuelen << 24); memcpy(pi->name, name, namelen + 1); memcpy(pi->value, value, valuelen + 1); - pa->toc[pa->count] = - (namelen << 24) | (((unsigned) pi) - ((unsigned) pa)); + pa->toc[pa->count] = (namelen << 24) | (((unsigned) pi) - ((unsigned) pa)); pa->count++; pa->serial++; @@ -403,7 +508,7 @@ unsigned int __system_property_serial(const prop_info *pi) unsigned int __system_property_wait_any(unsigned int serial) { - prop_area *pa = __system_property_area__; + prop_area *pa = __system_property_regions__[0]; do { __futex_wait(&pa->serial, serial, 0); diff --git a/libc/include/sys/_system_properties.h b/libc/include/sys/_system_properties.h index c5bc2235b..4971a4c12 100644 --- a/libc/include/sys/_system_properties.h +++ b/libc/include/sys/_system_properties.h @@ -42,12 +42,13 @@ typedef struct prop_msg prop_msg; #define PROP_SERVICE_NAME "property_service" #define PROP_FILENAME "/dev/__properties__" -/* (8 header words + 247 toc words) = 1020 bytes */ -/* 1024 bytes header and toc + 247 prop_infos @ 128 bytes = 32640 bytes */ +/* (4 header words + 28 toc words) = 128 bytes */ +/* 128 bytes header and toc + 28 prop_infos @ 128 bytes = 3712 bytes */ -#define PA_COUNT_MAX 247 -#define PA_INFO_START 1024 -#define PA_SIZE 32768 +#define PA_COUNT_MAX 28 +#define PA_REGION_COUNT 128 +#define PA_INFO_START 128 +#define PA_SIZE 4096 #define TOC_NAME_LEN(toc) ((toc) >> 24) #define TOC_TO_INFO(area, toc) ((prop_info*) (((char*) area) + ((toc) & 0xFFFFFF))) @@ -96,12 +97,18 @@ struct prop_msg #define PROP_PATH_LOCAL_OVERRIDE "/data/local.prop" #define PROP_PATH_FACTORY "/factory/factory.prop" +/* +** Map the property area from the specified filename. This +** method is for testing only. +*/ +int __system_property_set_filename(const char *filename); + /* ** Initialize the area to be used to store properties. Can ** only be done by a single process that has write access to ** the property area. */ -void __system_property_area_init(void *data); +int __system_property_area_init(); /* Add a new system property. Can only be done by a single ** process that has write access to the property area, and diff --git a/tests/property_benchmark.cpp b/tests/property_benchmark.cpp index 2c8e2a115..7266bd08a 100644 --- a/tests/property_benchmark.cpp +++ b/tests/property_benchmark.cpp @@ -15,23 +15,40 @@ */ #include "benchmark.h" +#include #define _REALLY_INCLUDE_SYS__SYSTEM_PROPERTIES_H_ #include #include +#include -extern void *__system_property_area__; +extern void *__system_property_regions__[PA_REGION_COUNT]; #define TEST_NUM_PROPS \ - Arg(1)->Arg(4)->Arg(16)->Arg(64)->Arg(128)->Arg(247) + Arg(1)->Arg(4)->Arg(16)->Arg(64)->Arg(128)->Arg(256)->Arg(512)->Arg(1024) struct LocalPropertyTestState { - LocalPropertyTestState(int nprops) : nprops(nprops) { + LocalPropertyTestState(int nprops) : nprops(nprops), valid(false) { static const char prop_name_chars[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-_"; - old_pa = __system_property_area__; - pa = malloc(PA_SIZE); - __system_property_area_init(pa); + + char dir_template[] = "/data/nativetest/prop-XXXXXX"; + char *dirname = mkdtemp(dir_template); + if (!dirname) { + perror("making temp file for test state failed (is /data/nativetest writable?)"); + return; + } + + for (size_t i = 0; i < PA_REGION_COUNT; i++) { + old_pa[i] = __system_property_regions__[i]; + __system_property_regions__[i] = NULL; + } + + pa_dirname = dirname; + pa_filename = pa_dirname + "/__properties__"; + + __system_property_set_filename(pa_filename.c_str()); + __system_property_area_init(); names = new char* [nprops]; name_lens = new int[nprops]; @@ -54,10 +71,22 @@ struct LocalPropertyTestState { } __system_property_add(names[i], name_lens[i], values[i], value_lens[i]); } + + valid = true; } ~LocalPropertyTestState() { - __system_property_area__ = old_pa; + if (!valid) + return; + + for (size_t i = 0; i < PA_REGION_COUNT; i++) { + __system_property_regions__[i] = old_pa[i]; + } + + __system_property_set_filename(PROP_FILENAME); + unlink(pa_filename.c_str()); + rmdir(pa_dirname.c_str()); + for (int i = 0; i < nprops; i++) { delete names[i]; delete values[i]; @@ -66,7 +95,6 @@ struct LocalPropertyTestState { delete[] name_lens; delete[] values; delete[] value_lens; - free(pa); } public: const int nprops; @@ -74,10 +102,12 @@ public: int *name_lens; char **values; int *value_lens; + bool valid; private: - void *pa; - void *old_pa; + std::string pa_dirname; + std::string pa_filename; + void *old_pa[PA_REGION_COUNT]; }; static void BM_property_get(int iters, int nprops) @@ -87,6 +117,9 @@ static void BM_property_get(int iters, int nprops) LocalPropertyTestState pa(nprops); char value[PROP_VALUE_MAX]; + if (!pa.valid) + return; + srandom(iters * nprops); StartBenchmarkTiming(); @@ -104,6 +137,9 @@ static void BM_property_find(int iters, int nprops) LocalPropertyTestState pa(nprops); + if (!pa.valid) + return; + srandom(iters * nprops); StartBenchmarkTiming(); diff --git a/tests/system_properties_test.cpp b/tests/system_properties_test.cpp index 70ff1d662..50bdfdfe4 100644 --- a/tests/system_properties_test.cpp +++ b/tests/system_properties_test.cpp @@ -16,32 +16,61 @@ #include #include +#include +#include #if __BIONIC__ #define _REALLY_INCLUDE_SYS__SYSTEM_PROPERTIES_H_ #include -extern void *__system_property_area__; +extern void *__system_property_regions__[PA_REGION_COUNT]; struct LocalPropertyTestState { - LocalPropertyTestState() { - old_pa = __system_property_area__; - pa = malloc(PA_SIZE); - __system_property_area_init(pa); + LocalPropertyTestState() : valid(false) { + char dir_template[] = "/data/nativetest/prop-XXXXXX"; + char *dirname = mkdtemp(dir_template); + if (!dirname) { + perror("making temp file for test state failed (is /data/nativetest writable?)"); + return; + } + + for (size_t i = 0; i < PA_REGION_COUNT; i++) { + old_pa[i] = __system_property_regions__[i]; + __system_property_regions__[i] = NULL; + } + + pa_dirname = dirname; + pa_filename = pa_dirname + "/__properties__"; + + __system_property_set_filename(pa_filename.c_str()); + __system_property_area_init(); + valid = true; } ~LocalPropertyTestState() { - __system_property_area__ = old_pa; - free(pa); + if (!valid) + return; + + for (size_t i = 0; i < PA_REGION_COUNT; i++) { + __system_property_regions__[i] = old_pa[i]; + } + + __system_property_set_filename(PROP_FILENAME); + unlink(pa_filename.c_str()); + rmdir(pa_dirname.c_str()); } +public: + bool valid; private: - void *pa; - void *old_pa; + std::string pa_dirname; + std::string pa_filename; + void *old_pa[PA_REGION_COUNT]; }; TEST(properties, add) { LocalPropertyTestState pa; + ASSERT_TRUE(pa.valid); char propvalue[PROP_VALUE_MAX]; @@ -61,6 +90,7 @@ TEST(properties, add) { TEST(properties, update) { LocalPropertyTestState pa; + ASSERT_TRUE(pa.valid); char propvalue[PROP_VALUE_MAX]; prop_info *pi; @@ -91,27 +121,34 @@ TEST(properties, update) { ASSERT_STREQ(propvalue, "value6"); } -// 247 = max # of properties supported by current implementation -// (this should never go down) -TEST(properties, fill_247) { +TEST(properties, fill) { LocalPropertyTestState pa; + ASSERT_TRUE(pa.valid); char prop_name[PROP_NAME_MAX]; char prop_value[PROP_VALUE_MAX]; char prop_value_ret[PROP_VALUE_MAX]; + int count = 0; int ret; - for (int i = 0; i < 247; i++) { - ret = snprintf(prop_name, PROP_NAME_MAX - 1, "property_%d", i); + while (true) { + ret = snprintf(prop_name, PROP_NAME_MAX - 1, "property_%d", count); memset(prop_name + ret, 'a', PROP_NAME_MAX - 1 - ret); - ret = snprintf(prop_value, PROP_VALUE_MAX - 1, "value_%d", i); + ret = snprintf(prop_value, PROP_VALUE_MAX - 1, "value_%d", count); memset(prop_value + ret, 'b', PROP_VALUE_MAX - 1 - ret); prop_name[PROP_NAME_MAX - 1] = 0; prop_value[PROP_VALUE_MAX - 1] = 0; - ASSERT_EQ(0, __system_property_add(prop_name, PROP_NAME_MAX - 1, prop_value, PROP_VALUE_MAX - 1)); + ret = __system_property_add(prop_name, PROP_NAME_MAX - 1, prop_value, PROP_VALUE_MAX - 1); + if (ret < 0) + break; + + count++; } - for (int i = 0; i < 247; i++) { + // For historical reasons at least 247 properties must be supported + ASSERT_GE(count, 247); + + for (int i = 0; i < count; i++) { ret = snprintf(prop_name, PROP_NAME_MAX - 1, "property_%d", i); memset(prop_name + ret, 'a', PROP_NAME_MAX - 1 - ret); ret = snprintf(prop_value, PROP_VALUE_MAX - 1, "value_%d", i); @@ -134,6 +171,7 @@ static void foreach_test_callback(const prop_info *pi, void* cookie) { TEST(properties, foreach) { LocalPropertyTestState pa; + ASSERT_TRUE(pa.valid); size_t count = 0; ASSERT_EQ(0, __system_property_add("property", 8, "value1", 6)); @@ -146,6 +184,7 @@ TEST(properties, foreach) { TEST(properties, find_nth) { LocalPropertyTestState pa; + ASSERT_TRUE(pa.valid); ASSERT_EQ(0, __system_property_add("property", 8, "value1", 6)); ASSERT_EQ(0, __system_property_add("other_property", 14, "value2", 6)); @@ -165,6 +204,7 @@ TEST(properties, find_nth) { TEST(properties, errors) { LocalPropertyTestState pa; + ASSERT_TRUE(pa.valid); char prop_value[PROP_NAME_MAX]; ASSERT_EQ(0, __system_property_add("property", 8, "value1", 6)); @@ -181,6 +221,7 @@ TEST(properties, errors) { TEST(properties, serial) { LocalPropertyTestState pa; + ASSERT_TRUE(pa.valid); const prop_info *pi; unsigned int serial; @@ -206,6 +247,7 @@ static void *PropertyWaitHelperFn(void *arg) TEST(properties, wait) { LocalPropertyTestState pa; + ASSERT_TRUE(pa.valid); unsigned int serial; prop_info *pi; pthread_t t; From 5bfa3ee8b37ef162154559575193018a6235acba Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Thu, 20 Jun 2013 10:33:28 -0700 Subject: [PATCH 23/77] bionic: add missing memory barriers to system properties 1) Reading the value must finish before checking whether it's intact 2) Setting the serial's dirty bit must visible before modifying the value 3) The modified value must be visible before clearing the serial's dirty bit 4) New properties and their TOC entries must be visible before updating the property count Change-Id: I26c680ec025fdb72362d5f618ec0d2b93d381233 Signed-off-by: Greg Hackmann --- libc/bionic/system_properties.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index 800c8b84f..b12879e47 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -49,6 +49,7 @@ #include #include +#include struct prop_area { unsigned volatile count; @@ -315,6 +316,7 @@ int __system_property_read(const prop_info *pi, char *name, char *value) } len = SERIAL_VALUE_LEN(serial); memcpy(value, pi->value, len + 1); + ANDROID_MEMBAR_FULL(); if(serial == pi->serial) { if(name != 0) { strcpy(name, pi->name); @@ -446,7 +448,9 @@ int __system_property_update(prop_info *pi, const char *value, unsigned int len) return -1; pi->serial = pi->serial | 1; + ANDROID_MEMBAR_FULL(); memcpy(pi->value, value, len + 1); + ANDROID_MEMBAR_FULL(); pi->serial = (len << 24) | ((pi->serial + 1) & 0xffffff); __futex_wake(&pi->serial, INT32_MAX); @@ -493,6 +497,7 @@ int __system_property_add(const char *name, unsigned int namelen, memcpy(pi->value, value, valuelen + 1); pa->toc[pa->count] = (namelen << 24) | (((unsigned) pi) - ((unsigned) pa)); + ANDROID_MEMBAR_FULL(); pa->count++; pa->serial++; From 6ac8e6a46d71a51bec16938efa89f275fa89cf7d Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Thu, 20 Jun 2013 11:27:56 -0700 Subject: [PATCH 24/77] bionic: reimplement property area as hybrid trie/binary tree See the comments for an explanation of how properties are stored. The trie structure is designed to scale better than the previous array-based implementation. Searching an array with n properties required average O(n) string compares of the entire key; searching the trie requires average O(log n) string compares of each token (substrings between '.' characters). Change-Id: I491305bc7aca59609abcd871a5f33d97f89ce714 Signed-off-by: Greg Hackmann --- libc/bionic/system_properties.c | 369 +++++++++++++++++++------- libc/include/sys/_system_properties.h | 12 +- 2 files changed, 280 insertions(+), 101 deletions(-) diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index b12879e47..126fea56a 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -34,6 +34,7 @@ #include #include #include +#include #include @@ -51,12 +52,15 @@ #include #include +#define ALIGN(x, a) (((x) + (a - 1)) & ~(a - 1)) + struct prop_area { - unsigned volatile count; + unsigned bytes_used; unsigned volatile serial; unsigned magic; unsigned version; - unsigned toc[1]; + unsigned reserved[28]; + char data[0]; }; typedef struct prop_area prop_area; @@ -69,11 +73,46 @@ struct prop_info { typedef struct prop_info prop_info; +/* + * Properties are stored in a hybrid trie/binary tree structure. + * Each property's name is delimited at '.' characters, and the tokens are put + * into a trie structure. Siblings at each level of the trie are stored in a + * binary tree. For instance, "ro.secure"="1" could be stored as follows: + * + * +-----+ children +----+ children +--------+ + * | |-------------->| ro |-------------->| secure | + * +-----+ +----+ +--------+ + * / \ / | + * left / \ right left / | prop +===========+ + * v v v +-------->| ro.secure | + * +-----+ +-----+ +-----+ +-----------+ + * | net | | sys | | com | | 1 | + * +-----+ +-----+ +-----+ +===========+ + */ + +typedef volatile uint32_t prop_off_t; +struct prop_bt { + char name[PROP_NAME_MAX]; + uint8_t namelen; + uint8_t reserved[3]; + + prop_off_t prop; + + prop_off_t left; + prop_off_t right; + + prop_off_t children; +}; + +typedef struct prop_bt prop_bt; + static const char property_service_socket[] = "/dev/socket/" PROP_SERVICE_NAME; static char property_filename[PATH_MAX] = PROP_FILENAME; prop_area *__system_property_regions__[PA_REGION_COUNT] = { NULL, }; +const size_t PA_DATA_SIZE = PA_SIZE - sizeof(prop_area); + static int get_fd_from_env(void) { char *env = getenv("ANDROID_PROPERTY_WORKSPACE"); @@ -120,6 +159,11 @@ static int map_prop_region_rw(size_t region) pa->magic = PROP_AREA_MAGIC; pa->version = PROP_AREA_VERSION; + if (!region) { + /* reserve root node */ + pa->bytes_used += sizeof(prop_bt); + } + /* plug into the lib property services */ __system_property_regions__[region] = pa; @@ -225,83 +269,183 @@ int __system_properties_init() return map_prop_region(0); } -int __system_property_foreach( - void (*propfn)(const prop_info *pi, void *cookie), - void *cookie) +static void *new_prop_obj(size_t size, prop_off_t *off) { - size_t region; + prop_area *pa; + size_t i, idx; + size = ALIGN(size, sizeof(uint32_t)); - for (region = 0; region < PA_REGION_COUNT; region++) { - prop_area *pa; - unsigned i; - - int err = map_prop_region(region); - if (err < 0) - break; - pa = __system_property_regions__[region]; - - for (i = 0; i < pa->count; i++) { - unsigned entry = pa->toc[i]; - prop_info *pi = TOC_TO_INFO(pa, entry); - propfn(pi, cookie); + for (i = 0; i < PA_REGION_COUNT; i++) { + int err = map_prop_region_rw(i); + if (err < 0) { + return NULL; } + + pa = __system_property_regions__[i]; + if (pa->bytes_used + size <= PA_DATA_SIZE) + break; } - return 0; + if (i == PA_REGION_COUNT) + return NULL; + + idx = pa->bytes_used; + *off = idx + i * PA_DATA_SIZE; + pa->bytes_used += size; + return pa->data + idx; } -const prop_info *__system_property_find_nth(unsigned n) +static prop_bt *new_prop_bt(const char *name, uint8_t namelen, prop_off_t *off) { - size_t region = n / PA_COUNT_MAX; - prop_area *pa; + prop_off_t off_tmp; + prop_bt *bt = new_prop_obj(sizeof(prop_bt), &off_tmp); + if (bt) { + memcpy(bt->name, name, namelen); + bt->name[namelen] = '\0'; + bt->namelen = namelen; + ANDROID_MEMBAR_FULL(); + *off = off_tmp; + } - int err = map_prop_region(region); - if (err < 0) + return bt; +} + +static prop_info *new_prop_info(const char *name, uint8_t namelen, + const char *value, uint8_t valuelen, prop_off_t *off) +{ + prop_off_t off_tmp; + prop_info *info = new_prop_obj(sizeof(prop_info), &off_tmp); + if (info) { + memcpy(info->name, name, namelen); + info->name[namelen] = '\0'; + info->serial = (valuelen << 24); + memcpy(info->value, value, valuelen); + info->value[valuelen] = '\0'; + ANDROID_MEMBAR_FULL(); + *off = off_tmp; + } + + return info; +} + +static void *to_prop_obj(prop_off_t off) +{ + size_t region = off / PA_DATA_SIZE; + size_t idx = off % PA_DATA_SIZE; + + if (region > PA_REGION_COUNT) return NULL; - pa = __system_property_regions__[region]; - if((n % PA_COUNT_MAX) >= pa->count) { - return 0; + if (map_prop_region(region) < 0) + return NULL; + + return __system_property_regions__[region]->data + idx; +} + +static prop_bt *root_node() +{ + return to_prop_obj(0); +} + +static int cmp_prop_name(const char *one, uint8_t one_len, const char *two, + uint8_t two_len) +{ + if (one_len < two_len) + return -1; + else if (one_len > two_len) + return 1; + else + return strncmp(one, two, one_len); +} + +static prop_bt *find_prop_bt(prop_bt *bt, const char *name, uint8_t namelen, + bool alloc_if_needed) +{ + while (true) { + int ret; + if (!bt) + return bt; + ret = cmp_prop_name(name, namelen, bt->name, bt->namelen); + + if (ret == 0) { + return bt; + } else if (ret < 0) { + if (bt->left) { + bt = to_prop_obj(bt->left); + } else { + if (!alloc_if_needed) + return NULL; + + bt = new_prop_bt(name, namelen, &bt->left); + } + } else { + if (bt->right) { + bt = to_prop_obj(bt->right); + } else { + if (!alloc_if_needed) + return NULL; + + bt = new_prop_bt(name, namelen, &bt->right); + } + } + } +} + +static const prop_info *find_property(prop_bt *trie, const char *name, + uint8_t namelen, const char *value, uint8_t valuelen, + bool alloc_if_needed) +{ + const char *remaining_name = name; + + while (true) { + char *sep = strchr(remaining_name, '.'); + bool want_subtree = (sep != NULL); + uint8_t substr_size; + + prop_bt *root; + + if (want_subtree) { + substr_size = sep - remaining_name; + } else { + substr_size = strlen(remaining_name); + } + + if (!substr_size) + return NULL; + + if (trie->children) { + root = to_prop_obj(trie->children); + } else if (alloc_if_needed) { + root = new_prop_bt(remaining_name, substr_size, &trie->children); + } else { + root = NULL; + } + + if (!root) + return NULL; + + trie = find_prop_bt(root, remaining_name, substr_size, alloc_if_needed); + if (!trie) + return NULL; + + if (!want_subtree) + break; + + remaining_name = sep + 1; + } + + if (trie->prop) { + return to_prop_obj(trie->prop); + } else if (alloc_if_needed) { + return new_prop_info(name, namelen, value, valuelen, &trie->prop); } else { - return TOC_TO_INFO(pa, pa->toc[n]); + return NULL; } } const prop_info *__system_property_find(const char *name) { - unsigned len = strlen(name); - size_t region; - - if (len >= PROP_NAME_MAX) - return 0; - if (len < 1) - return 0; - - for (region = 0; region < PA_REGION_COUNT; region++) { - prop_area *pa; - unsigned count; - unsigned *toc; - prop_info *pi; - - int err = map_prop_region(region); - if (err < 0) - return 0; - pa = __system_property_regions__[region]; - count = pa->count; - toc = pa->toc; - - while(count--) { - unsigned entry = *toc++; - if(TOC_NAME_LEN(entry) != len) continue; - - pi = TOC_TO_INFO(pa, entry); - if(memcmp(name, pi->name, len)) continue; - - return pi; - } - } - - return 0; + return find_property(root_node(), name, strlen(name), NULL, 0, false); } int __system_property_read(const prop_info *pi, char *name, char *value) @@ -463,10 +607,8 @@ int __system_property_update(prop_info *pi, const char *value, unsigned int len) int __system_property_add(const char *name, unsigned int namelen, const char *value, unsigned int valuelen) { - prop_area *pa; - prop_info *pa_info_array; - prop_info *pi; - size_t region; + prop_area *pa = __system_property_regions__[0]; + const prop_info *pi; if (namelen >= PROP_NAME_MAX) return -1; @@ -475,34 +617,12 @@ int __system_property_add(const char *name, unsigned int namelen, if (namelen < 1) return -1; - for (region = 0; region < PA_REGION_COUNT; region++) - { - int err = map_prop_region_rw(region); - if (err < 0) - return -1; - - pa = __system_property_regions__[region]; - - if (pa->count < PA_COUNT_MAX) - break; - } - - if (region == PA_REGION_COUNT) + pi = find_property(root_node(), name, namelen, value, valuelen, true); + if (!pi) return -1; - pa_info_array = (void*) (((char*) pa) + PA_INFO_START); - pi = pa_info_array + pa->count; - pi->serial = (valuelen << 24); - memcpy(pi->name, name, namelen + 1); - memcpy(pi->value, value, valuelen + 1); - - pa->toc[pa->count] = (namelen << 24) | (((unsigned) pi) - ((unsigned) pa)); - ANDROID_MEMBAR_FULL(); - - pa->count++; pa->serial++; __futex_wake(&pa->serial, INT32_MAX); - return 0; } @@ -521,3 +641,72 @@ unsigned int __system_property_wait_any(unsigned int serial) return pa->serial; } + +struct find_nth_cookie { + unsigned count; + unsigned n; + const prop_info *pi; +}; + +static void find_nth_fn(const prop_info *pi, void *ptr) +{ + struct find_nth_cookie *cookie = ptr; + + if (cookie->n == cookie->count) + cookie->pi = pi; + + cookie->count++; +} + +const prop_info *__system_property_find_nth(unsigned n) +{ + struct find_nth_cookie cookie; + int err; + + memset(&cookie, 0, sizeof(cookie)); + cookie.n = n; + + err = __system_property_foreach(find_nth_fn, &cookie); + if (err < 0) + return NULL; + + return cookie.pi; +} + +static int foreach_property(prop_off_t off, + void (*propfn)(const prop_info *pi, void *cookie), void *cookie) +{ + prop_bt *trie = to_prop_obj(off); + if (!trie) + return -1; + + if (trie->left) { + int err = foreach_property(trie->left, propfn, cookie); + if (err < 0) + return -1; + } + if (trie->prop) { + prop_info *info = to_prop_obj(trie->prop); + if (!info) + return -1; + propfn(info, cookie); + } + if (trie->children) { + int err = foreach_property(trie->children, propfn, cookie); + if (err < 0) + return -1; + } + if (trie->right) { + int err = foreach_property(trie->right, propfn, cookie); + if (err < 0) + return -1; + } + + return 0; +} + +int __system_property_foreach(void (*propfn)(const prop_info *pi, void *cookie), + void *cookie) +{ + return foreach_property(0, propfn, cookie); +} diff --git a/libc/include/sys/_system_properties.h b/libc/include/sys/_system_properties.h index 4971a4c12..9c48e1b3e 100644 --- a/libc/include/sys/_system_properties.h +++ b/libc/include/sys/_system_properties.h @@ -37,7 +37,7 @@ typedef struct prop_msg prop_msg; #define PROP_AREA_MAGIC 0x504f5250 -#define PROP_AREA_VERSION 0x45434f76 +#define PROP_AREA_VERSION 0xfc6ed0ab #define PROP_SERVICE_NAME "property_service" #define PROP_FILENAME "/dev/__properties__" @@ -45,14 +45,9 @@ typedef struct prop_msg prop_msg; /* (4 header words + 28 toc words) = 128 bytes */ /* 128 bytes header and toc + 28 prop_infos @ 128 bytes = 3712 bytes */ -#define PA_COUNT_MAX 28 #define PA_REGION_COUNT 128 -#define PA_INFO_START 128 #define PA_SIZE 4096 -#define TOC_NAME_LEN(toc) ((toc) >> 24) -#define TOC_TO_INFO(area, toc) ((prop_info*) (((char*) area) + ((toc) & 0xFFFFFF))) - #define SERIAL_VALUE_LEN(serial) ((serial) >> 24) #define SERIAL_DIRTY(serial) ((serial) & 1) @@ -84,11 +79,6 @@ struct prop_msg ** 1. pi->serial = pi->serial | 1 ** 2. memcpy(pi->value, local_value, value_len) ** 3. pi->serial = (value_len << 24) | ((pi->serial + 1) & 0xffffff) -** -** Improvements: -** - maintain the toc sorted by pi->name to allow lookup -** by binary search -** */ #define PROP_PATH_RAMDISK_DEFAULT "/default.prop" From 0f76e2340ca53ce458a01f91627651dfe0112969 Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Mon, 17 Jun 2013 12:37:09 -0700 Subject: [PATCH 25/77] bionic: add hierarchical properties test Deliberately put items several levels deep in the trie hierarchy to test the trie traversal Change-Id: Id3cbd2e7d3500216b1ac8025eac70c0939622903 Signed-off-by: Greg Hackmann --- tests/system_properties_test.cpp | 78 ++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/tests/system_properties_test.cpp b/tests/system_properties_test.cpp index 50bdfdfe4..bafd2c3e1 100644 --- a/tests/system_properties_test.cpp +++ b/tests/system_properties_test.cpp @@ -202,6 +202,84 @@ TEST(properties, find_nth) { ASSERT_EQ((const prop_info *)NULL, __system_property_find_nth(247)); } +static void hierarchical_test_callback(const prop_info *pi, void *cookie) { + bool (*ok)[8][8] = static_cast(cookie); + + char name[PROP_NAME_MAX]; + char value[PROP_VALUE_MAX]; + + __system_property_read(pi, name, value); + + int name_i, name_j, name_k; + int value_i, value_j, value_k; + ASSERT_EQ(3, sscanf(name, "property_%d.%d.%d", &name_i, &name_j, &name_k)); + ASSERT_EQ(3, sscanf(value, "value_%d.%d.%d", &value_i, &value_j, &value_k)); + ASSERT_EQ(name_i, value_i); + ASSERT_GE(name_i, 0); + ASSERT_LT(name_i, 8); + ASSERT_EQ(name_j, value_j); + ASSERT_GE(name_j, 0); + ASSERT_LT(name_j, 8); + ASSERT_EQ(name_k, value_k); + ASSERT_GE(name_k, 0); + ASSERT_LT(name_k, 8); + + ok[name_i][name_j][name_k] = true; +} + +TEST(properties, fill_hierarchical) { + LocalPropertyTestState pa; + ASSERT_TRUE(pa.valid); + char prop_name[PROP_NAME_MAX]; + char prop_value[PROP_VALUE_MAX]; + char prop_value_ret[PROP_VALUE_MAX]; + int ret; + + for (int i = 0; i < 8; i++) { + for (int j = 0; j < 8; j++) { + for (int k = 0; k < 8; k++) { + ret = snprintf(prop_name, PROP_NAME_MAX - 1, "property_%d.%d.%d", i, j, k); + memset(prop_name + ret, 'a', PROP_NAME_MAX - 1 - ret); + ret = snprintf(prop_value, PROP_VALUE_MAX - 1, "value_%d.%d.%d", i, j, k); + memset(prop_value + ret, 'b', PROP_VALUE_MAX - 1 - ret); + prop_name[PROP_NAME_MAX - 1] = 0; + prop_value[PROP_VALUE_MAX - 1] = 0; + + ASSERT_EQ(0, __system_property_add(prop_name, PROP_NAME_MAX - 1, prop_value, PROP_VALUE_MAX - 1)); + } + } + } + + for (int i = 0; i < 8; i++) { + for (int j = 0; j < 8; j++) { + for (int k = 0; k < 8; k++) { + ret = snprintf(prop_name, PROP_NAME_MAX - 1, "property_%d.%d.%d", i, j, k); + memset(prop_name + ret, 'a', PROP_NAME_MAX - 1 - ret); + ret = snprintf(prop_value, PROP_VALUE_MAX - 1, "value_%d.%d.%d", i, j, k); + memset(prop_value + ret, 'b', PROP_VALUE_MAX - 1 - ret); + prop_name[PROP_NAME_MAX - 1] = 0; + prop_value[PROP_VALUE_MAX - 1] = 0; + memset(prop_value_ret, '\0', PROP_VALUE_MAX); + + ASSERT_EQ(PROP_VALUE_MAX - 1, __system_property_get(prop_name, prop_value_ret)); + ASSERT_EQ(0, memcmp(prop_value, prop_value_ret, PROP_VALUE_MAX)); + } + } + } + + bool ok[8][8][8]; + memset(ok, 0, sizeof(ok)); + __system_property_foreach(hierarchical_test_callback, ok); + + for (int i = 0; i < 8; i++) { + for (int j = 0; j < 8; j++) { + for (int k = 0; k < 8; k++) { + ASSERT_TRUE(ok[i][j][k]); + } + } + } +} + TEST(properties, errors) { LocalPropertyTestState pa; ASSERT_TRUE(pa.valid); From 5f05348c18286a2cea46eae8acf94ed5b7932fac Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Wed, 19 Jun 2013 13:31:21 -0700 Subject: [PATCH 26/77] bionic: revert to a single (larger) property area d329697 is too complicated. Change the multiple property pages back to a single 128K property area that's mapped in entirely at initialization (the memory will not get allocated until the pages are touched). d329697 has other changes useful for testing (moving property area initialization inside bionic and adding __system_property_set_filename) so undo the change manually rather than with git revert. Change-Id: I0ecb27843404f93af5489f15bfe657d65175e4f0 Signed-off-by: Greg Hackmann --- libc/bionic/system_properties.c | 91 ++++++++------------------- libc/include/sys/_system_properties.h | 6 +- tests/property_benchmark.cpp | 14 ++--- tests/system_properties_test.cpp | 14 ++--- 4 files changed, 36 insertions(+), 89 deletions(-) diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index 126fea56a..d4054d245 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -109,7 +109,7 @@ typedef struct prop_bt prop_bt; static const char property_service_socket[] = "/dev/socket/" PROP_SERVICE_NAME; static char property_filename[PATH_MAX] = PROP_FILENAME; -prop_area *__system_property_regions__[PA_REGION_COUNT] = { NULL, }; +prop_area *__system_property_area__ = NULL; const size_t PA_DATA_SIZE = PA_SIZE - sizeof(prop_area); @@ -124,15 +124,10 @@ static int get_fd_from_env(void) return atoi(env); } -static int map_prop_region_rw(size_t region) +static int map_prop_area_rw() { prop_area *pa; int fd; - size_t offset = region * PA_SIZE; - - if (__system_property_regions__[region]) { - return 0; - } /* dev is a tmpfs that we can use to carve a shared workspace * out of, so let's do that... @@ -148,24 +143,21 @@ static int map_prop_region_rw(size_t region) return -1; } - if (ftruncate(fd, offset + PA_SIZE) < 0) + if (ftruncate(fd, PA_SIZE) < 0) goto out; - pa = mmap(NULL, PA_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, offset); + pa = mmap(NULL, PA_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if(pa == MAP_FAILED) goto out; memset(pa, 0, PA_SIZE); pa->magic = PROP_AREA_MAGIC; pa->version = PROP_AREA_VERSION; - - if (!region) { - /* reserve root node */ - pa->bytes_used += sizeof(prop_bt); - } + /* reserve root node */ + pa->bytes_used = sizeof(prop_bt); /* plug into the lib property services */ - __system_property_regions__[region] = pa; + __system_property_area__ = pa; close(fd); return 0; @@ -187,20 +179,14 @@ int __system_property_set_filename(const char *filename) int __system_property_area_init() { - return map_prop_region_rw(0); + return map_prop_area_rw(); } -static int map_prop_region(size_t region) +static int map_prop_area() { bool fromFile = true; - bool swapped; - size_t offset = region * PA_SIZE; int result = -1; - if(__system_property_regions__[region]) { - return 0; - } - int fd = open(property_filename, O_RDONLY | O_NOFOLLOW); if ((fd < 0) && (errno == ENOENT)) { @@ -229,11 +215,11 @@ static int map_prop_region(size_t region) if ((fd_stat.st_uid != 0) || (fd_stat.st_gid != 0) || ((fd_stat.st_mode & (S_IWGRP | S_IWOTH)) != 0) - || (fd_stat.st_size < offset + PA_SIZE) ) { + || (fd_stat.st_size < PA_SIZE) ) { goto cleanup; } - prop_area *pa = mmap(NULL, PA_SIZE, PROT_READ, MAP_SHARED, fd, offset); + prop_area *pa = mmap(NULL, PA_SIZE, PROT_READ, MAP_SHARED, fd, 0); if (pa == MAP_FAILED) { goto cleanup; @@ -245,16 +231,8 @@ static int map_prop_region(size_t region) } result = 0; - swapped = __sync_bool_compare_and_swap(&__system_property_regions__[region], - NULL, pa); - if (!swapped) { - /** - * In the event of a race either mapping is equally good, so - * the thread that lost can just throw its mapping away and proceed as - * normal. - */ - munmap(pa, PA_SIZE); - } + + __system_property_area__ = pa; cleanup: if (fromFile) { @@ -266,33 +244,20 @@ cleanup: int __system_properties_init() { - return map_prop_region(0); + return map_prop_area(); } static void *new_prop_obj(size_t size, prop_off_t *off) { - prop_area *pa; - size_t i, idx; + prop_area *pa = __system_property_area__; size = ALIGN(size, sizeof(uint32_t)); - for (i = 0; i < PA_REGION_COUNT; i++) { - int err = map_prop_region_rw(i); - if (err < 0) { - return NULL; - } - - pa = __system_property_regions__[i]; - if (pa->bytes_used + size <= PA_DATA_SIZE) - break; - } - - if (i == PA_REGION_COUNT) + if (pa->bytes_used + size > PA_DATA_SIZE) return NULL; - idx = pa->bytes_used; - *off = idx + i * PA_DATA_SIZE; - pa->bytes_used += size; - return pa->data + idx; + *off = pa->bytes_used; + __system_property_area__->bytes_used += size; + return __system_property_area__->data + *off; } static prop_bt *new_prop_bt(const char *name, uint8_t namelen, prop_off_t *off) @@ -330,16 +295,10 @@ static prop_info *new_prop_info(const char *name, uint8_t namelen, static void *to_prop_obj(prop_off_t off) { - size_t region = off / PA_DATA_SIZE; - size_t idx = off % PA_DATA_SIZE; - - if (region > PA_REGION_COUNT) + if (off > PA_DATA_SIZE) return NULL; - if (map_prop_region(region) < 0) - return NULL; - - return __system_property_regions__[region]->data + idx; + return __system_property_area__->data + off; } static prop_bt *root_node() @@ -570,7 +529,7 @@ int __system_property_wait(const prop_info *pi) { unsigned n; if(pi == 0) { - prop_area *pa = __system_property_regions__[0]; + prop_area *pa = __system_property_area__; n = pa->serial; do { __futex_wait(&pa->serial, n, 0); @@ -586,7 +545,7 @@ int __system_property_wait(const prop_info *pi) int __system_property_update(prop_info *pi, const char *value, unsigned int len) { - prop_area *pa = __system_property_regions__[0]; + prop_area *pa = __system_property_area__; if (len >= PROP_VALUE_MAX) return -1; @@ -607,7 +566,7 @@ int __system_property_update(prop_info *pi, const char *value, unsigned int len) int __system_property_add(const char *name, unsigned int namelen, const char *value, unsigned int valuelen) { - prop_area *pa = __system_property_regions__[0]; + prop_area *pa = __system_property_area__; const prop_info *pi; if (namelen >= PROP_NAME_MAX) @@ -633,7 +592,7 @@ unsigned int __system_property_serial(const prop_info *pi) unsigned int __system_property_wait_any(unsigned int serial) { - prop_area *pa = __system_property_regions__[0]; + prop_area *pa = __system_property_area__; do { __futex_wait(&pa->serial, serial, 0); diff --git a/libc/include/sys/_system_properties.h b/libc/include/sys/_system_properties.h index 9c48e1b3e..92e35e124 100644 --- a/libc/include/sys/_system_properties.h +++ b/libc/include/sys/_system_properties.h @@ -42,11 +42,7 @@ typedef struct prop_msg prop_msg; #define PROP_SERVICE_NAME "property_service" #define PROP_FILENAME "/dev/__properties__" -/* (4 header words + 28 toc words) = 128 bytes */ -/* 128 bytes header and toc + 28 prop_infos @ 128 bytes = 3712 bytes */ - -#define PA_REGION_COUNT 128 -#define PA_SIZE 4096 +#define PA_SIZE (128 * 1024) #define SERIAL_VALUE_LEN(serial) ((serial) >> 24) #define SERIAL_DIRTY(serial) ((serial) & 1) diff --git a/tests/property_benchmark.cpp b/tests/property_benchmark.cpp index 7266bd08a..d10be915e 100644 --- a/tests/property_benchmark.cpp +++ b/tests/property_benchmark.cpp @@ -23,7 +23,7 @@ #include #include -extern void *__system_property_regions__[PA_REGION_COUNT]; +extern void *__system_property_area__; #define TEST_NUM_PROPS \ Arg(1)->Arg(4)->Arg(16)->Arg(64)->Arg(128)->Arg(256)->Arg(512)->Arg(1024) @@ -39,10 +39,8 @@ struct LocalPropertyTestState { return; } - for (size_t i = 0; i < PA_REGION_COUNT; i++) { - old_pa[i] = __system_property_regions__[i]; - __system_property_regions__[i] = NULL; - } + old_pa = __system_property_area__; + __system_property_area__ = NULL; pa_dirname = dirname; pa_filename = pa_dirname + "/__properties__"; @@ -79,9 +77,7 @@ struct LocalPropertyTestState { if (!valid) return; - for (size_t i = 0; i < PA_REGION_COUNT; i++) { - __system_property_regions__[i] = old_pa[i]; - } + __system_property_area__ = old_pa; __system_property_set_filename(PROP_FILENAME); unlink(pa_filename.c_str()); @@ -107,7 +103,7 @@ public: private: std::string pa_dirname; std::string pa_filename; - void *old_pa[PA_REGION_COUNT]; + void *old_pa; }; static void BM_property_get(int iters, int nprops) diff --git a/tests/system_properties_test.cpp b/tests/system_properties_test.cpp index bafd2c3e1..b9256c664 100644 --- a/tests/system_properties_test.cpp +++ b/tests/system_properties_test.cpp @@ -24,7 +24,7 @@ #define _REALLY_INCLUDE_SYS__SYSTEM_PROPERTIES_H_ #include -extern void *__system_property_regions__[PA_REGION_COUNT]; +extern void *__system_property_area__; struct LocalPropertyTestState { LocalPropertyTestState() : valid(false) { @@ -35,10 +35,8 @@ struct LocalPropertyTestState { return; } - for (size_t i = 0; i < PA_REGION_COUNT; i++) { - old_pa[i] = __system_property_regions__[i]; - __system_property_regions__[i] = NULL; - } + old_pa = __system_property_area__; + __system_property_area__ = NULL; pa_dirname = dirname; pa_filename = pa_dirname + "/__properties__"; @@ -52,9 +50,7 @@ struct LocalPropertyTestState { if (!valid) return; - for (size_t i = 0; i < PA_REGION_COUNT; i++) { - __system_property_regions__[i] = old_pa[i]; - } + __system_property_area__ = old_pa; __system_property_set_filename(PROP_FILENAME); unlink(pa_filename.c_str()); @@ -65,7 +61,7 @@ public: private: std::string pa_dirname; std::string pa_filename; - void *old_pa[PA_REGION_COUNT]; + void *old_pa; }; TEST(properties, add) { From fb9b7b436f3ef94385f1b0c55ab81f246f0d96b8 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Sun, 16 Jun 2013 10:19:16 -0700 Subject: [PATCH 27/77] bionic: prevent root processes from calling __system_property_add If a root process other than init calls __system_property_add, which it should never do, it will break the design assumption that there is only one mutator. Pass O_EXCL to open() in map_prop_region_rw to ensure that only one process ever has the property pages open for write. Change-Id: I6b7c118e5e2fd2b92a2b168b8454fe9707325462 --- libc/bionic/system_properties.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index d4054d245..f9671c636 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -128,11 +128,13 @@ static int map_prop_area_rw() { prop_area *pa; int fd; + int ret; /* dev is a tmpfs that we can use to carve a shared workspace * out of, so let's do that... */ - fd = open(property_filename, O_RDWR | O_CREAT | O_NOFOLLOW, 0644); + fd = open(property_filename, O_RDWR | O_CREAT | O_NOFOLLOW | O_CLOEXEC | + O_EXCL, 0444); if (fd < 0) { if (errno == EACCES) { /* for consistency with the case where the process has already @@ -143,6 +145,10 @@ static int map_prop_area_rw() return -1; } + ret = fcntl(fd, F_SETFD, FD_CLOEXEC); + if (ret < 0) + goto out; + if (ftruncate(fd, PA_SIZE) < 0) goto out; @@ -186,8 +192,16 @@ static int map_prop_area() { bool fromFile = true; int result = -1; + int fd; + int ret; - int fd = open(property_filename, O_RDONLY | O_NOFOLLOW); + fd = open(property_filename, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (fd >= 0) { + /* For old kernels that don't support O_CLOEXEC */ + ret = fcntl(fd, F_SETFD, FD_CLOEXEC); + if (ret < 0) + goto cleanup; + } if ((fd < 0) && (errno == ENOENT)) { /* From 492ce95d9f6149137cb5b63c55cf2b3cdbe51e5e Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Fri, 21 Jun 2013 13:02:38 -0700 Subject: [PATCH 28/77] bionic: store property names as variable-length strings Names are immutable, so the fixed-sized arrays can be replaced with variable-length ones to save memory (especially on internal tree nodes). Change-Id: Iee77874b4b55b0f9c5e531d1334be7f7f40086d2 Signed-off-by: Greg Hackmann --- libc/bionic/system_properties.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index f9671c636..481e6ae45 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -66,9 +66,9 @@ struct prop_area { typedef struct prop_area prop_area; struct prop_info { - char name[PROP_NAME_MAX]; unsigned volatile serial; char value[PROP_VALUE_MAX]; + char name[0]; }; typedef struct prop_info prop_info; @@ -92,7 +92,6 @@ typedef struct prop_info prop_info; typedef volatile uint32_t prop_off_t; struct prop_bt { - char name[PROP_NAME_MAX]; uint8_t namelen; uint8_t reserved[3]; @@ -102,6 +101,8 @@ struct prop_bt { prop_off_t right; prop_off_t children; + + char name[0]; }; typedef struct prop_bt prop_bt; @@ -277,7 +278,7 @@ static void *new_prop_obj(size_t size, prop_off_t *off) static prop_bt *new_prop_bt(const char *name, uint8_t namelen, prop_off_t *off) { prop_off_t off_tmp; - prop_bt *bt = new_prop_obj(sizeof(prop_bt), &off_tmp); + prop_bt *bt = new_prop_obj(sizeof(prop_bt) + namelen + 1, &off_tmp); if (bt) { memcpy(bt->name, name, namelen); bt->name[namelen] = '\0'; @@ -293,7 +294,7 @@ static prop_info *new_prop_info(const char *name, uint8_t namelen, const char *value, uint8_t valuelen, prop_off_t *off) { prop_off_t off_tmp; - prop_info *info = new_prop_obj(sizeof(prop_info), &off_tmp); + prop_info *info = new_prop_obj(sizeof(prop_info) + namelen + 1, &off_tmp); if (info) { memcpy(info->name, name, namelen); info->name[namelen] = '\0'; From 982463d99b56a53d6b560a12fe19ed4c7675b425 Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Mon, 24 Jun 2013 18:05:10 -0700 Subject: [PATCH 29/77] Revert "bionic: store property names as variable-length strings" This reverts commit 492ce95d9f6149137cb5b63c55cf2b3cdbe51e5e. --- libc/bionic/system_properties.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index 481e6ae45..f9671c636 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -66,9 +66,9 @@ struct prop_area { typedef struct prop_area prop_area; struct prop_info { + char name[PROP_NAME_MAX]; unsigned volatile serial; char value[PROP_VALUE_MAX]; - char name[0]; }; typedef struct prop_info prop_info; @@ -92,6 +92,7 @@ typedef struct prop_info prop_info; typedef volatile uint32_t prop_off_t; struct prop_bt { + char name[PROP_NAME_MAX]; uint8_t namelen; uint8_t reserved[3]; @@ -101,8 +102,6 @@ struct prop_bt { prop_off_t right; prop_off_t children; - - char name[0]; }; typedef struct prop_bt prop_bt; @@ -278,7 +277,7 @@ static void *new_prop_obj(size_t size, prop_off_t *off) static prop_bt *new_prop_bt(const char *name, uint8_t namelen, prop_off_t *off) { prop_off_t off_tmp; - prop_bt *bt = new_prop_obj(sizeof(prop_bt) + namelen + 1, &off_tmp); + prop_bt *bt = new_prop_obj(sizeof(prop_bt), &off_tmp); if (bt) { memcpy(bt->name, name, namelen); bt->name[namelen] = '\0'; @@ -294,7 +293,7 @@ static prop_info *new_prop_info(const char *name, uint8_t namelen, const char *value, uint8_t valuelen, prop_off_t *off) { prop_off_t off_tmp; - prop_info *info = new_prop_obj(sizeof(prop_info) + namelen + 1, &off_tmp); + prop_info *info = new_prop_obj(sizeof(prop_info), &off_tmp); if (info) { memcpy(info->name, name, namelen); info->name[namelen] = '\0'; From b938756eb3a2911b7230abf0b35356146afd19dc Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Mon, 24 Jun 2013 18:05:14 -0700 Subject: [PATCH 30/77] Revert "bionic: prevent root processes from calling __system_property_add" This reverts commit fb9b7b436f3ef94385f1b0c55ab81f246f0d96b8. --- libc/bionic/system_properties.c | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index f9671c636..d4054d245 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -128,13 +128,11 @@ static int map_prop_area_rw() { prop_area *pa; int fd; - int ret; /* dev is a tmpfs that we can use to carve a shared workspace * out of, so let's do that... */ - fd = open(property_filename, O_RDWR | O_CREAT | O_NOFOLLOW | O_CLOEXEC | - O_EXCL, 0444); + fd = open(property_filename, O_RDWR | O_CREAT | O_NOFOLLOW, 0644); if (fd < 0) { if (errno == EACCES) { /* for consistency with the case where the process has already @@ -145,10 +143,6 @@ static int map_prop_area_rw() return -1; } - ret = fcntl(fd, F_SETFD, FD_CLOEXEC); - if (ret < 0) - goto out; - if (ftruncate(fd, PA_SIZE) < 0) goto out; @@ -192,16 +186,8 @@ static int map_prop_area() { bool fromFile = true; int result = -1; - int fd; - int ret; - fd = open(property_filename, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); - if (fd >= 0) { - /* For old kernels that don't support O_CLOEXEC */ - ret = fcntl(fd, F_SETFD, FD_CLOEXEC); - if (ret < 0) - goto cleanup; - } + int fd = open(property_filename, O_RDONLY | O_NOFOLLOW); if ((fd < 0) && (errno == ENOENT)) { /* From fa8cc0629f6227b507434245d237d44d7e119b16 Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Mon, 24 Jun 2013 18:05:16 -0700 Subject: [PATCH 31/77] Revert "bionic: revert to a single (larger) property area" This reverts commit 5f05348c18286a2cea46eae8acf94ed5b7932fac. --- libc/bionic/system_properties.c | 91 +++++++++++++++++++-------- libc/include/sys/_system_properties.h | 6 +- tests/property_benchmark.cpp | 14 +++-- tests/system_properties_test.cpp | 14 +++-- 4 files changed, 89 insertions(+), 36 deletions(-) diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index d4054d245..126fea56a 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -109,7 +109,7 @@ typedef struct prop_bt prop_bt; static const char property_service_socket[] = "/dev/socket/" PROP_SERVICE_NAME; static char property_filename[PATH_MAX] = PROP_FILENAME; -prop_area *__system_property_area__ = NULL; +prop_area *__system_property_regions__[PA_REGION_COUNT] = { NULL, }; const size_t PA_DATA_SIZE = PA_SIZE - sizeof(prop_area); @@ -124,10 +124,15 @@ static int get_fd_from_env(void) return atoi(env); } -static int map_prop_area_rw() +static int map_prop_region_rw(size_t region) { prop_area *pa; int fd; + size_t offset = region * PA_SIZE; + + if (__system_property_regions__[region]) { + return 0; + } /* dev is a tmpfs that we can use to carve a shared workspace * out of, so let's do that... @@ -143,21 +148,24 @@ static int map_prop_area_rw() return -1; } - if (ftruncate(fd, PA_SIZE) < 0) + if (ftruncate(fd, offset + PA_SIZE) < 0) goto out; - pa = mmap(NULL, PA_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + pa = mmap(NULL, PA_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, offset); if(pa == MAP_FAILED) goto out; memset(pa, 0, PA_SIZE); pa->magic = PROP_AREA_MAGIC; pa->version = PROP_AREA_VERSION; - /* reserve root node */ - pa->bytes_used = sizeof(prop_bt); + + if (!region) { + /* reserve root node */ + pa->bytes_used += sizeof(prop_bt); + } /* plug into the lib property services */ - __system_property_area__ = pa; + __system_property_regions__[region] = pa; close(fd); return 0; @@ -179,14 +187,20 @@ int __system_property_set_filename(const char *filename) int __system_property_area_init() { - return map_prop_area_rw(); + return map_prop_region_rw(0); } -static int map_prop_area() +static int map_prop_region(size_t region) { bool fromFile = true; + bool swapped; + size_t offset = region * PA_SIZE; int result = -1; + if(__system_property_regions__[region]) { + return 0; + } + int fd = open(property_filename, O_RDONLY | O_NOFOLLOW); if ((fd < 0) && (errno == ENOENT)) { @@ -215,11 +229,11 @@ static int map_prop_area() if ((fd_stat.st_uid != 0) || (fd_stat.st_gid != 0) || ((fd_stat.st_mode & (S_IWGRP | S_IWOTH)) != 0) - || (fd_stat.st_size < PA_SIZE) ) { + || (fd_stat.st_size < offset + PA_SIZE) ) { goto cleanup; } - prop_area *pa = mmap(NULL, PA_SIZE, PROT_READ, MAP_SHARED, fd, 0); + prop_area *pa = mmap(NULL, PA_SIZE, PROT_READ, MAP_SHARED, fd, offset); if (pa == MAP_FAILED) { goto cleanup; @@ -231,8 +245,16 @@ static int map_prop_area() } result = 0; - - __system_property_area__ = pa; + swapped = __sync_bool_compare_and_swap(&__system_property_regions__[region], + NULL, pa); + if (!swapped) { + /** + * In the event of a race either mapping is equally good, so + * the thread that lost can just throw its mapping away and proceed as + * normal. + */ + munmap(pa, PA_SIZE); + } cleanup: if (fromFile) { @@ -244,20 +266,33 @@ cleanup: int __system_properties_init() { - return map_prop_area(); + return map_prop_region(0); } static void *new_prop_obj(size_t size, prop_off_t *off) { - prop_area *pa = __system_property_area__; + prop_area *pa; + size_t i, idx; size = ALIGN(size, sizeof(uint32_t)); - if (pa->bytes_used + size > PA_DATA_SIZE) + for (i = 0; i < PA_REGION_COUNT; i++) { + int err = map_prop_region_rw(i); + if (err < 0) { + return NULL; + } + + pa = __system_property_regions__[i]; + if (pa->bytes_used + size <= PA_DATA_SIZE) + break; + } + + if (i == PA_REGION_COUNT) return NULL; - *off = pa->bytes_used; - __system_property_area__->bytes_used += size; - return __system_property_area__->data + *off; + idx = pa->bytes_used; + *off = idx + i * PA_DATA_SIZE; + pa->bytes_used += size; + return pa->data + idx; } static prop_bt *new_prop_bt(const char *name, uint8_t namelen, prop_off_t *off) @@ -295,10 +330,16 @@ static prop_info *new_prop_info(const char *name, uint8_t namelen, static void *to_prop_obj(prop_off_t off) { - if (off > PA_DATA_SIZE) + size_t region = off / PA_DATA_SIZE; + size_t idx = off % PA_DATA_SIZE; + + if (region > PA_REGION_COUNT) return NULL; - return __system_property_area__->data + off; + if (map_prop_region(region) < 0) + return NULL; + + return __system_property_regions__[region]->data + idx; } static prop_bt *root_node() @@ -529,7 +570,7 @@ int __system_property_wait(const prop_info *pi) { unsigned n; if(pi == 0) { - prop_area *pa = __system_property_area__; + prop_area *pa = __system_property_regions__[0]; n = pa->serial; do { __futex_wait(&pa->serial, n, 0); @@ -545,7 +586,7 @@ int __system_property_wait(const prop_info *pi) int __system_property_update(prop_info *pi, const char *value, unsigned int len) { - prop_area *pa = __system_property_area__; + prop_area *pa = __system_property_regions__[0]; if (len >= PROP_VALUE_MAX) return -1; @@ -566,7 +607,7 @@ int __system_property_update(prop_info *pi, const char *value, unsigned int len) int __system_property_add(const char *name, unsigned int namelen, const char *value, unsigned int valuelen) { - prop_area *pa = __system_property_area__; + prop_area *pa = __system_property_regions__[0]; const prop_info *pi; if (namelen >= PROP_NAME_MAX) @@ -592,7 +633,7 @@ unsigned int __system_property_serial(const prop_info *pi) unsigned int __system_property_wait_any(unsigned int serial) { - prop_area *pa = __system_property_area__; + prop_area *pa = __system_property_regions__[0]; do { __futex_wait(&pa->serial, serial, 0); diff --git a/libc/include/sys/_system_properties.h b/libc/include/sys/_system_properties.h index 92e35e124..9c48e1b3e 100644 --- a/libc/include/sys/_system_properties.h +++ b/libc/include/sys/_system_properties.h @@ -42,7 +42,11 @@ typedef struct prop_msg prop_msg; #define PROP_SERVICE_NAME "property_service" #define PROP_FILENAME "/dev/__properties__" -#define PA_SIZE (128 * 1024) +/* (4 header words + 28 toc words) = 128 bytes */ +/* 128 bytes header and toc + 28 prop_infos @ 128 bytes = 3712 bytes */ + +#define PA_REGION_COUNT 128 +#define PA_SIZE 4096 #define SERIAL_VALUE_LEN(serial) ((serial) >> 24) #define SERIAL_DIRTY(serial) ((serial) & 1) diff --git a/tests/property_benchmark.cpp b/tests/property_benchmark.cpp index d10be915e..7266bd08a 100644 --- a/tests/property_benchmark.cpp +++ b/tests/property_benchmark.cpp @@ -23,7 +23,7 @@ #include #include -extern void *__system_property_area__; +extern void *__system_property_regions__[PA_REGION_COUNT]; #define TEST_NUM_PROPS \ Arg(1)->Arg(4)->Arg(16)->Arg(64)->Arg(128)->Arg(256)->Arg(512)->Arg(1024) @@ -39,8 +39,10 @@ struct LocalPropertyTestState { return; } - old_pa = __system_property_area__; - __system_property_area__ = NULL; + for (size_t i = 0; i < PA_REGION_COUNT; i++) { + old_pa[i] = __system_property_regions__[i]; + __system_property_regions__[i] = NULL; + } pa_dirname = dirname; pa_filename = pa_dirname + "/__properties__"; @@ -77,7 +79,9 @@ struct LocalPropertyTestState { if (!valid) return; - __system_property_area__ = old_pa; + for (size_t i = 0; i < PA_REGION_COUNT; i++) { + __system_property_regions__[i] = old_pa[i]; + } __system_property_set_filename(PROP_FILENAME); unlink(pa_filename.c_str()); @@ -103,7 +107,7 @@ public: private: std::string pa_dirname; std::string pa_filename; - void *old_pa; + void *old_pa[PA_REGION_COUNT]; }; static void BM_property_get(int iters, int nprops) diff --git a/tests/system_properties_test.cpp b/tests/system_properties_test.cpp index b9256c664..bafd2c3e1 100644 --- a/tests/system_properties_test.cpp +++ b/tests/system_properties_test.cpp @@ -24,7 +24,7 @@ #define _REALLY_INCLUDE_SYS__SYSTEM_PROPERTIES_H_ #include -extern void *__system_property_area__; +extern void *__system_property_regions__[PA_REGION_COUNT]; struct LocalPropertyTestState { LocalPropertyTestState() : valid(false) { @@ -35,8 +35,10 @@ struct LocalPropertyTestState { return; } - old_pa = __system_property_area__; - __system_property_area__ = NULL; + for (size_t i = 0; i < PA_REGION_COUNT; i++) { + old_pa[i] = __system_property_regions__[i]; + __system_property_regions__[i] = NULL; + } pa_dirname = dirname; pa_filename = pa_dirname + "/__properties__"; @@ -50,7 +52,9 @@ struct LocalPropertyTestState { if (!valid) return; - __system_property_area__ = old_pa; + for (size_t i = 0; i < PA_REGION_COUNT; i++) { + __system_property_regions__[i] = old_pa[i]; + } __system_property_set_filename(PROP_FILENAME); unlink(pa_filename.c_str()); @@ -61,7 +65,7 @@ public: private: std::string pa_dirname; std::string pa_filename; - void *old_pa; + void *old_pa[PA_REGION_COUNT]; }; TEST(properties, add) { From 890aa0d937ae72473c84cc773e77d45e9288b0bc Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Mon, 24 Jun 2013 18:05:18 -0700 Subject: [PATCH 32/77] Revert "bionic: add hierarchical properties test" This reverts commit 0f76e2340ca53ce458a01f91627651dfe0112969. --- tests/system_properties_test.cpp | 78 -------------------------------- 1 file changed, 78 deletions(-) diff --git a/tests/system_properties_test.cpp b/tests/system_properties_test.cpp index bafd2c3e1..50bdfdfe4 100644 --- a/tests/system_properties_test.cpp +++ b/tests/system_properties_test.cpp @@ -202,84 +202,6 @@ TEST(properties, find_nth) { ASSERT_EQ((const prop_info *)NULL, __system_property_find_nth(247)); } -static void hierarchical_test_callback(const prop_info *pi, void *cookie) { - bool (*ok)[8][8] = static_cast(cookie); - - char name[PROP_NAME_MAX]; - char value[PROP_VALUE_MAX]; - - __system_property_read(pi, name, value); - - int name_i, name_j, name_k; - int value_i, value_j, value_k; - ASSERT_EQ(3, sscanf(name, "property_%d.%d.%d", &name_i, &name_j, &name_k)); - ASSERT_EQ(3, sscanf(value, "value_%d.%d.%d", &value_i, &value_j, &value_k)); - ASSERT_EQ(name_i, value_i); - ASSERT_GE(name_i, 0); - ASSERT_LT(name_i, 8); - ASSERT_EQ(name_j, value_j); - ASSERT_GE(name_j, 0); - ASSERT_LT(name_j, 8); - ASSERT_EQ(name_k, value_k); - ASSERT_GE(name_k, 0); - ASSERT_LT(name_k, 8); - - ok[name_i][name_j][name_k] = true; -} - -TEST(properties, fill_hierarchical) { - LocalPropertyTestState pa; - ASSERT_TRUE(pa.valid); - char prop_name[PROP_NAME_MAX]; - char prop_value[PROP_VALUE_MAX]; - char prop_value_ret[PROP_VALUE_MAX]; - int ret; - - for (int i = 0; i < 8; i++) { - for (int j = 0; j < 8; j++) { - for (int k = 0; k < 8; k++) { - ret = snprintf(prop_name, PROP_NAME_MAX - 1, "property_%d.%d.%d", i, j, k); - memset(prop_name + ret, 'a', PROP_NAME_MAX - 1 - ret); - ret = snprintf(prop_value, PROP_VALUE_MAX - 1, "value_%d.%d.%d", i, j, k); - memset(prop_value + ret, 'b', PROP_VALUE_MAX - 1 - ret); - prop_name[PROP_NAME_MAX - 1] = 0; - prop_value[PROP_VALUE_MAX - 1] = 0; - - ASSERT_EQ(0, __system_property_add(prop_name, PROP_NAME_MAX - 1, prop_value, PROP_VALUE_MAX - 1)); - } - } - } - - for (int i = 0; i < 8; i++) { - for (int j = 0; j < 8; j++) { - for (int k = 0; k < 8; k++) { - ret = snprintf(prop_name, PROP_NAME_MAX - 1, "property_%d.%d.%d", i, j, k); - memset(prop_name + ret, 'a', PROP_NAME_MAX - 1 - ret); - ret = snprintf(prop_value, PROP_VALUE_MAX - 1, "value_%d.%d.%d", i, j, k); - memset(prop_value + ret, 'b', PROP_VALUE_MAX - 1 - ret); - prop_name[PROP_NAME_MAX - 1] = 0; - prop_value[PROP_VALUE_MAX - 1] = 0; - memset(prop_value_ret, '\0', PROP_VALUE_MAX); - - ASSERT_EQ(PROP_VALUE_MAX - 1, __system_property_get(prop_name, prop_value_ret)); - ASSERT_EQ(0, memcmp(prop_value, prop_value_ret, PROP_VALUE_MAX)); - } - } - } - - bool ok[8][8][8]; - memset(ok, 0, sizeof(ok)); - __system_property_foreach(hierarchical_test_callback, ok); - - for (int i = 0; i < 8; i++) { - for (int j = 0; j < 8; j++) { - for (int k = 0; k < 8; k++) { - ASSERT_TRUE(ok[i][j][k]); - } - } - } -} - TEST(properties, errors) { LocalPropertyTestState pa; ASSERT_TRUE(pa.valid); From 44c273c2933bb6550b0fa12bfe218594f0780e25 Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Mon, 24 Jun 2013 18:05:20 -0700 Subject: [PATCH 33/77] Revert "bionic: reimplement property area as hybrid trie/binary tree" This reverts commit 6ac8e6a46d71a51bec16938efa89f275fa89cf7d. --- libc/bionic/system_properties.c | 381 +++++++------------------- libc/include/sys/_system_properties.h | 12 +- 2 files changed, 107 insertions(+), 286 deletions(-) diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index 126fea56a..b12879e47 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -34,7 +34,6 @@ #include #include #include -#include #include @@ -52,15 +51,12 @@ #include #include -#define ALIGN(x, a) (((x) + (a - 1)) & ~(a - 1)) - struct prop_area { - unsigned bytes_used; + unsigned volatile count; unsigned volatile serial; unsigned magic; unsigned version; - unsigned reserved[28]; - char data[0]; + unsigned toc[1]; }; typedef struct prop_area prop_area; @@ -73,46 +69,11 @@ struct prop_info { typedef struct prop_info prop_info; -/* - * Properties are stored in a hybrid trie/binary tree structure. - * Each property's name is delimited at '.' characters, and the tokens are put - * into a trie structure. Siblings at each level of the trie are stored in a - * binary tree. For instance, "ro.secure"="1" could be stored as follows: - * - * +-----+ children +----+ children +--------+ - * | |-------------->| ro |-------------->| secure | - * +-----+ +----+ +--------+ - * / \ / | - * left / \ right left / | prop +===========+ - * v v v +-------->| ro.secure | - * +-----+ +-----+ +-----+ +-----------+ - * | net | | sys | | com | | 1 | - * +-----+ +-----+ +-----+ +===========+ - */ - -typedef volatile uint32_t prop_off_t; -struct prop_bt { - char name[PROP_NAME_MAX]; - uint8_t namelen; - uint8_t reserved[3]; - - prop_off_t prop; - - prop_off_t left; - prop_off_t right; - - prop_off_t children; -}; - -typedef struct prop_bt prop_bt; - static const char property_service_socket[] = "/dev/socket/" PROP_SERVICE_NAME; static char property_filename[PATH_MAX] = PROP_FILENAME; prop_area *__system_property_regions__[PA_REGION_COUNT] = { NULL, }; -const size_t PA_DATA_SIZE = PA_SIZE - sizeof(prop_area); - static int get_fd_from_env(void) { char *env = getenv("ANDROID_PROPERTY_WORKSPACE"); @@ -159,11 +120,6 @@ static int map_prop_region_rw(size_t region) pa->magic = PROP_AREA_MAGIC; pa->version = PROP_AREA_VERSION; - if (!region) { - /* reserve root node */ - pa->bytes_used += sizeof(prop_bt); - } - /* plug into the lib property services */ __system_property_regions__[region] = pa; @@ -269,183 +225,83 @@ int __system_properties_init() return map_prop_region(0); } -static void *new_prop_obj(size_t size, prop_off_t *off) +int __system_property_foreach( + void (*propfn)(const prop_info *pi, void *cookie), + void *cookie) { + size_t region; + + for (region = 0; region < PA_REGION_COUNT; region++) { + prop_area *pa; + unsigned i; + + int err = map_prop_region(region); + if (err < 0) + break; + pa = __system_property_regions__[region]; + + for (i = 0; i < pa->count; i++) { + unsigned entry = pa->toc[i]; + prop_info *pi = TOC_TO_INFO(pa, entry); + propfn(pi, cookie); + } + } + + return 0; +} + +const prop_info *__system_property_find_nth(unsigned n) +{ + size_t region = n / PA_COUNT_MAX; prop_area *pa; - size_t i, idx; - size = ALIGN(size, sizeof(uint32_t)); - for (i = 0; i < PA_REGION_COUNT; i++) { - int err = map_prop_region_rw(i); - if (err < 0) { - return NULL; - } - - pa = __system_property_regions__[i]; - if (pa->bytes_used + size <= PA_DATA_SIZE) - break; - } - - if (i == PA_REGION_COUNT) + int err = map_prop_region(region); + if (err < 0) return NULL; + pa = __system_property_regions__[region]; - idx = pa->bytes_used; - *off = idx + i * PA_DATA_SIZE; - pa->bytes_used += size; - return pa->data + idx; -} - -static prop_bt *new_prop_bt(const char *name, uint8_t namelen, prop_off_t *off) -{ - prop_off_t off_tmp; - prop_bt *bt = new_prop_obj(sizeof(prop_bt), &off_tmp); - if (bt) { - memcpy(bt->name, name, namelen); - bt->name[namelen] = '\0'; - bt->namelen = namelen; - ANDROID_MEMBAR_FULL(); - *off = off_tmp; - } - - return bt; -} - -static prop_info *new_prop_info(const char *name, uint8_t namelen, - const char *value, uint8_t valuelen, prop_off_t *off) -{ - prop_off_t off_tmp; - prop_info *info = new_prop_obj(sizeof(prop_info), &off_tmp); - if (info) { - memcpy(info->name, name, namelen); - info->name[namelen] = '\0'; - info->serial = (valuelen << 24); - memcpy(info->value, value, valuelen); - info->value[valuelen] = '\0'; - ANDROID_MEMBAR_FULL(); - *off = off_tmp; - } - - return info; -} - -static void *to_prop_obj(prop_off_t off) -{ - size_t region = off / PA_DATA_SIZE; - size_t idx = off % PA_DATA_SIZE; - - if (region > PA_REGION_COUNT) - return NULL; - - if (map_prop_region(region) < 0) - return NULL; - - return __system_property_regions__[region]->data + idx; -} - -static prop_bt *root_node() -{ - return to_prop_obj(0); -} - -static int cmp_prop_name(const char *one, uint8_t one_len, const char *two, - uint8_t two_len) -{ - if (one_len < two_len) - return -1; - else if (one_len > two_len) - return 1; - else - return strncmp(one, two, one_len); -} - -static prop_bt *find_prop_bt(prop_bt *bt, const char *name, uint8_t namelen, - bool alloc_if_needed) -{ - while (true) { - int ret; - if (!bt) - return bt; - ret = cmp_prop_name(name, namelen, bt->name, bt->namelen); - - if (ret == 0) { - return bt; - } else if (ret < 0) { - if (bt->left) { - bt = to_prop_obj(bt->left); - } else { - if (!alloc_if_needed) - return NULL; - - bt = new_prop_bt(name, namelen, &bt->left); - } - } else { - if (bt->right) { - bt = to_prop_obj(bt->right); - } else { - if (!alloc_if_needed) - return NULL; - - bt = new_prop_bt(name, namelen, &bt->right); - } - } - } -} - -static const prop_info *find_property(prop_bt *trie, const char *name, - uint8_t namelen, const char *value, uint8_t valuelen, - bool alloc_if_needed) -{ - const char *remaining_name = name; - - while (true) { - char *sep = strchr(remaining_name, '.'); - bool want_subtree = (sep != NULL); - uint8_t substr_size; - - prop_bt *root; - - if (want_subtree) { - substr_size = sep - remaining_name; - } else { - substr_size = strlen(remaining_name); - } - - if (!substr_size) - return NULL; - - if (trie->children) { - root = to_prop_obj(trie->children); - } else if (alloc_if_needed) { - root = new_prop_bt(remaining_name, substr_size, &trie->children); - } else { - root = NULL; - } - - if (!root) - return NULL; - - trie = find_prop_bt(root, remaining_name, substr_size, alloc_if_needed); - if (!trie) - return NULL; - - if (!want_subtree) - break; - - remaining_name = sep + 1; - } - - if (trie->prop) { - return to_prop_obj(trie->prop); - } else if (alloc_if_needed) { - return new_prop_info(name, namelen, value, valuelen, &trie->prop); + if((n % PA_COUNT_MAX) >= pa->count) { + return 0; } else { - return NULL; + return TOC_TO_INFO(pa, pa->toc[n]); } } const prop_info *__system_property_find(const char *name) { - return find_property(root_node(), name, strlen(name), NULL, 0, false); + unsigned len = strlen(name); + size_t region; + + if (len >= PROP_NAME_MAX) + return 0; + if (len < 1) + return 0; + + for (region = 0; region < PA_REGION_COUNT; region++) { + prop_area *pa; + unsigned count; + unsigned *toc; + prop_info *pi; + + int err = map_prop_region(region); + if (err < 0) + return 0; + pa = __system_property_regions__[region]; + count = pa->count; + toc = pa->toc; + + while(count--) { + unsigned entry = *toc++; + if(TOC_NAME_LEN(entry) != len) continue; + + pi = TOC_TO_INFO(pa, entry); + if(memcmp(name, pi->name, len)) continue; + + return pi; + } + } + + return 0; } int __system_property_read(const prop_info *pi, char *name, char *value) @@ -607,8 +463,10 @@ int __system_property_update(prop_info *pi, const char *value, unsigned int len) int __system_property_add(const char *name, unsigned int namelen, const char *value, unsigned int valuelen) { - prop_area *pa = __system_property_regions__[0]; - const prop_info *pi; + prop_area *pa; + prop_info *pa_info_array; + prop_info *pi; + size_t region; if (namelen >= PROP_NAME_MAX) return -1; @@ -617,12 +475,34 @@ int __system_property_add(const char *name, unsigned int namelen, if (namelen < 1) return -1; - pi = find_property(root_node(), name, namelen, value, valuelen, true); - if (!pi) + for (region = 0; region < PA_REGION_COUNT; region++) + { + int err = map_prop_region_rw(region); + if (err < 0) + return -1; + + pa = __system_property_regions__[region]; + + if (pa->count < PA_COUNT_MAX) + break; + } + + if (region == PA_REGION_COUNT) return -1; + pa_info_array = (void*) (((char*) pa) + PA_INFO_START); + pi = pa_info_array + pa->count; + pi->serial = (valuelen << 24); + memcpy(pi->name, name, namelen + 1); + memcpy(pi->value, value, valuelen + 1); + + pa->toc[pa->count] = (namelen << 24) | (((unsigned) pi) - ((unsigned) pa)); + ANDROID_MEMBAR_FULL(); + + pa->count++; pa->serial++; __futex_wake(&pa->serial, INT32_MAX); + return 0; } @@ -641,72 +521,3 @@ unsigned int __system_property_wait_any(unsigned int serial) return pa->serial; } - -struct find_nth_cookie { - unsigned count; - unsigned n; - const prop_info *pi; -}; - -static void find_nth_fn(const prop_info *pi, void *ptr) -{ - struct find_nth_cookie *cookie = ptr; - - if (cookie->n == cookie->count) - cookie->pi = pi; - - cookie->count++; -} - -const prop_info *__system_property_find_nth(unsigned n) -{ - struct find_nth_cookie cookie; - int err; - - memset(&cookie, 0, sizeof(cookie)); - cookie.n = n; - - err = __system_property_foreach(find_nth_fn, &cookie); - if (err < 0) - return NULL; - - return cookie.pi; -} - -static int foreach_property(prop_off_t off, - void (*propfn)(const prop_info *pi, void *cookie), void *cookie) -{ - prop_bt *trie = to_prop_obj(off); - if (!trie) - return -1; - - if (trie->left) { - int err = foreach_property(trie->left, propfn, cookie); - if (err < 0) - return -1; - } - if (trie->prop) { - prop_info *info = to_prop_obj(trie->prop); - if (!info) - return -1; - propfn(info, cookie); - } - if (trie->children) { - int err = foreach_property(trie->children, propfn, cookie); - if (err < 0) - return -1; - } - if (trie->right) { - int err = foreach_property(trie->right, propfn, cookie); - if (err < 0) - return -1; - } - - return 0; -} - -int __system_property_foreach(void (*propfn)(const prop_info *pi, void *cookie), - void *cookie) -{ - return foreach_property(0, propfn, cookie); -} diff --git a/libc/include/sys/_system_properties.h b/libc/include/sys/_system_properties.h index 9c48e1b3e..4971a4c12 100644 --- a/libc/include/sys/_system_properties.h +++ b/libc/include/sys/_system_properties.h @@ -37,7 +37,7 @@ typedef struct prop_msg prop_msg; #define PROP_AREA_MAGIC 0x504f5250 -#define PROP_AREA_VERSION 0xfc6ed0ab +#define PROP_AREA_VERSION 0x45434f76 #define PROP_SERVICE_NAME "property_service" #define PROP_FILENAME "/dev/__properties__" @@ -45,9 +45,14 @@ typedef struct prop_msg prop_msg; /* (4 header words + 28 toc words) = 128 bytes */ /* 128 bytes header and toc + 28 prop_infos @ 128 bytes = 3712 bytes */ +#define PA_COUNT_MAX 28 #define PA_REGION_COUNT 128 +#define PA_INFO_START 128 #define PA_SIZE 4096 +#define TOC_NAME_LEN(toc) ((toc) >> 24) +#define TOC_TO_INFO(area, toc) ((prop_info*) (((char*) area) + ((toc) & 0xFFFFFF))) + #define SERIAL_VALUE_LEN(serial) ((serial) >> 24) #define SERIAL_DIRTY(serial) ((serial) & 1) @@ -79,6 +84,11 @@ struct prop_msg ** 1. pi->serial = pi->serial | 1 ** 2. memcpy(pi->value, local_value, value_len) ** 3. pi->serial = (value_len << 24) | ((pi->serial + 1) & 0xffffff) +** +** Improvements: +** - maintain the toc sorted by pi->name to allow lookup +** by binary search +** */ #define PROP_PATH_RAMDISK_DEFAULT "/default.prop" From de76bafb21477fbfac903abdc8154925da94b21e Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Mon, 24 Jun 2013 18:41:46 -0700 Subject: [PATCH 34/77] bionic: replace 4K property pages with 32K property pages Fixes apps compiled statically against new libc but running with old init Bug: 9558625 Change-Id: I79e6f02575d278d4c7d8e8546d772ed0529bcaab Signed-off-by: Greg Hackmann --- libc/include/sys/_system_properties.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libc/include/sys/_system_properties.h b/libc/include/sys/_system_properties.h index 4971a4c12..702a4b265 100644 --- a/libc/include/sys/_system_properties.h +++ b/libc/include/sys/_system_properties.h @@ -45,10 +45,10 @@ typedef struct prop_msg prop_msg; /* (4 header words + 28 toc words) = 128 bytes */ /* 128 bytes header and toc + 28 prop_infos @ 128 bytes = 3712 bytes */ -#define PA_COUNT_MAX 28 -#define PA_REGION_COUNT 128 -#define PA_INFO_START 128 -#define PA_SIZE 4096 +#define PA_COUNT_MAX 247 +#define PA_REGION_COUNT 16 +#define PA_INFO_START 1024 +#define PA_SIZE 32768 #define TOC_NAME_LEN(toc) ((toc) >> 24) #define TOC_TO_INFO(area, toc) ((prop_info*) (((char*) area) + ((toc) & 0xFFFFFF))) From c79de97cccbe695cd37d3be1985c8e0e5100d51d Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Mon, 24 Jun 2013 18:41:46 -0700 Subject: [PATCH 35/77] bionic: replace 4K property pages with 32K property pages Fixes apps compiled statically against new libc but running with old init Bug: 9558625 Change-Id: I79e6f02575d278d4c7d8e8546d772ed0529bcaab Signed-off-by: Greg Hackmann --- libc/include/sys/_system_properties.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libc/include/sys/_system_properties.h b/libc/include/sys/_system_properties.h index 4971a4c12..702a4b265 100644 --- a/libc/include/sys/_system_properties.h +++ b/libc/include/sys/_system_properties.h @@ -45,10 +45,10 @@ typedef struct prop_msg prop_msg; /* (4 header words + 28 toc words) = 128 bytes */ /* 128 bytes header and toc + 28 prop_infos @ 128 bytes = 3712 bytes */ -#define PA_COUNT_MAX 28 -#define PA_REGION_COUNT 128 -#define PA_INFO_START 128 -#define PA_SIZE 4096 +#define PA_COUNT_MAX 247 +#define PA_REGION_COUNT 16 +#define PA_INFO_START 1024 +#define PA_SIZE 32768 #define TOC_NAME_LEN(toc) ((toc) >> 24) #define TOC_TO_INFO(area, toc) ((prop_info*) (((char*) area) + ((toc) & 0xFFFFFF))) From 4482d8513920d614e3799a214c5bad07b72580d2 Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Thu, 20 Jun 2013 11:27:56 -0700 Subject: [PATCH 36/77] bionic: reimplement property area as hybrid trie/binary tree See the comments for an explanation of how properties are stored. The trie structure is designed to scale better than the previous array-based implementation. Searching an array with n properties required average O(n) string compares of the entire key; searching the trie requires average O(log n) string compares of each token (substrings between '.' characters). Change-Id: Ic28b3c5246004a3c502eb02d6e1c9b512884d872 Signed-off-by: Greg Hackmann --- libc/bionic/system_properties.c | 369 +++++++++++++++++++------- libc/include/sys/_system_properties.h | 16 +- 2 files changed, 282 insertions(+), 103 deletions(-) diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index b12879e47..126fea56a 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -34,6 +34,7 @@ #include #include #include +#include #include @@ -51,12 +52,15 @@ #include #include +#define ALIGN(x, a) (((x) + (a - 1)) & ~(a - 1)) + struct prop_area { - unsigned volatile count; + unsigned bytes_used; unsigned volatile serial; unsigned magic; unsigned version; - unsigned toc[1]; + unsigned reserved[28]; + char data[0]; }; typedef struct prop_area prop_area; @@ -69,11 +73,46 @@ struct prop_info { typedef struct prop_info prop_info; +/* + * Properties are stored in a hybrid trie/binary tree structure. + * Each property's name is delimited at '.' characters, and the tokens are put + * into a trie structure. Siblings at each level of the trie are stored in a + * binary tree. For instance, "ro.secure"="1" could be stored as follows: + * + * +-----+ children +----+ children +--------+ + * | |-------------->| ro |-------------->| secure | + * +-----+ +----+ +--------+ + * / \ / | + * left / \ right left / | prop +===========+ + * v v v +-------->| ro.secure | + * +-----+ +-----+ +-----+ +-----------+ + * | net | | sys | | com | | 1 | + * +-----+ +-----+ +-----+ +===========+ + */ + +typedef volatile uint32_t prop_off_t; +struct prop_bt { + char name[PROP_NAME_MAX]; + uint8_t namelen; + uint8_t reserved[3]; + + prop_off_t prop; + + prop_off_t left; + prop_off_t right; + + prop_off_t children; +}; + +typedef struct prop_bt prop_bt; + static const char property_service_socket[] = "/dev/socket/" PROP_SERVICE_NAME; static char property_filename[PATH_MAX] = PROP_FILENAME; prop_area *__system_property_regions__[PA_REGION_COUNT] = { NULL, }; +const size_t PA_DATA_SIZE = PA_SIZE - sizeof(prop_area); + static int get_fd_from_env(void) { char *env = getenv("ANDROID_PROPERTY_WORKSPACE"); @@ -120,6 +159,11 @@ static int map_prop_region_rw(size_t region) pa->magic = PROP_AREA_MAGIC; pa->version = PROP_AREA_VERSION; + if (!region) { + /* reserve root node */ + pa->bytes_used += sizeof(prop_bt); + } + /* plug into the lib property services */ __system_property_regions__[region] = pa; @@ -225,83 +269,183 @@ int __system_properties_init() return map_prop_region(0); } -int __system_property_foreach( - void (*propfn)(const prop_info *pi, void *cookie), - void *cookie) +static void *new_prop_obj(size_t size, prop_off_t *off) { - size_t region; + prop_area *pa; + size_t i, idx; + size = ALIGN(size, sizeof(uint32_t)); - for (region = 0; region < PA_REGION_COUNT; region++) { - prop_area *pa; - unsigned i; - - int err = map_prop_region(region); - if (err < 0) - break; - pa = __system_property_regions__[region]; - - for (i = 0; i < pa->count; i++) { - unsigned entry = pa->toc[i]; - prop_info *pi = TOC_TO_INFO(pa, entry); - propfn(pi, cookie); + for (i = 0; i < PA_REGION_COUNT; i++) { + int err = map_prop_region_rw(i); + if (err < 0) { + return NULL; } + + pa = __system_property_regions__[i]; + if (pa->bytes_used + size <= PA_DATA_SIZE) + break; } - return 0; + if (i == PA_REGION_COUNT) + return NULL; + + idx = pa->bytes_used; + *off = idx + i * PA_DATA_SIZE; + pa->bytes_used += size; + return pa->data + idx; } -const prop_info *__system_property_find_nth(unsigned n) +static prop_bt *new_prop_bt(const char *name, uint8_t namelen, prop_off_t *off) { - size_t region = n / PA_COUNT_MAX; - prop_area *pa; + prop_off_t off_tmp; + prop_bt *bt = new_prop_obj(sizeof(prop_bt), &off_tmp); + if (bt) { + memcpy(bt->name, name, namelen); + bt->name[namelen] = '\0'; + bt->namelen = namelen; + ANDROID_MEMBAR_FULL(); + *off = off_tmp; + } - int err = map_prop_region(region); - if (err < 0) + return bt; +} + +static prop_info *new_prop_info(const char *name, uint8_t namelen, + const char *value, uint8_t valuelen, prop_off_t *off) +{ + prop_off_t off_tmp; + prop_info *info = new_prop_obj(sizeof(prop_info), &off_tmp); + if (info) { + memcpy(info->name, name, namelen); + info->name[namelen] = '\0'; + info->serial = (valuelen << 24); + memcpy(info->value, value, valuelen); + info->value[valuelen] = '\0'; + ANDROID_MEMBAR_FULL(); + *off = off_tmp; + } + + return info; +} + +static void *to_prop_obj(prop_off_t off) +{ + size_t region = off / PA_DATA_SIZE; + size_t idx = off % PA_DATA_SIZE; + + if (region > PA_REGION_COUNT) return NULL; - pa = __system_property_regions__[region]; - if((n % PA_COUNT_MAX) >= pa->count) { - return 0; + if (map_prop_region(region) < 0) + return NULL; + + return __system_property_regions__[region]->data + idx; +} + +static prop_bt *root_node() +{ + return to_prop_obj(0); +} + +static int cmp_prop_name(const char *one, uint8_t one_len, const char *two, + uint8_t two_len) +{ + if (one_len < two_len) + return -1; + else if (one_len > two_len) + return 1; + else + return strncmp(one, two, one_len); +} + +static prop_bt *find_prop_bt(prop_bt *bt, const char *name, uint8_t namelen, + bool alloc_if_needed) +{ + while (true) { + int ret; + if (!bt) + return bt; + ret = cmp_prop_name(name, namelen, bt->name, bt->namelen); + + if (ret == 0) { + return bt; + } else if (ret < 0) { + if (bt->left) { + bt = to_prop_obj(bt->left); + } else { + if (!alloc_if_needed) + return NULL; + + bt = new_prop_bt(name, namelen, &bt->left); + } + } else { + if (bt->right) { + bt = to_prop_obj(bt->right); + } else { + if (!alloc_if_needed) + return NULL; + + bt = new_prop_bt(name, namelen, &bt->right); + } + } + } +} + +static const prop_info *find_property(prop_bt *trie, const char *name, + uint8_t namelen, const char *value, uint8_t valuelen, + bool alloc_if_needed) +{ + const char *remaining_name = name; + + while (true) { + char *sep = strchr(remaining_name, '.'); + bool want_subtree = (sep != NULL); + uint8_t substr_size; + + prop_bt *root; + + if (want_subtree) { + substr_size = sep - remaining_name; + } else { + substr_size = strlen(remaining_name); + } + + if (!substr_size) + return NULL; + + if (trie->children) { + root = to_prop_obj(trie->children); + } else if (alloc_if_needed) { + root = new_prop_bt(remaining_name, substr_size, &trie->children); + } else { + root = NULL; + } + + if (!root) + return NULL; + + trie = find_prop_bt(root, remaining_name, substr_size, alloc_if_needed); + if (!trie) + return NULL; + + if (!want_subtree) + break; + + remaining_name = sep + 1; + } + + if (trie->prop) { + return to_prop_obj(trie->prop); + } else if (alloc_if_needed) { + return new_prop_info(name, namelen, value, valuelen, &trie->prop); } else { - return TOC_TO_INFO(pa, pa->toc[n]); + return NULL; } } const prop_info *__system_property_find(const char *name) { - unsigned len = strlen(name); - size_t region; - - if (len >= PROP_NAME_MAX) - return 0; - if (len < 1) - return 0; - - for (region = 0; region < PA_REGION_COUNT; region++) { - prop_area *pa; - unsigned count; - unsigned *toc; - prop_info *pi; - - int err = map_prop_region(region); - if (err < 0) - return 0; - pa = __system_property_regions__[region]; - count = pa->count; - toc = pa->toc; - - while(count--) { - unsigned entry = *toc++; - if(TOC_NAME_LEN(entry) != len) continue; - - pi = TOC_TO_INFO(pa, entry); - if(memcmp(name, pi->name, len)) continue; - - return pi; - } - } - - return 0; + return find_property(root_node(), name, strlen(name), NULL, 0, false); } int __system_property_read(const prop_info *pi, char *name, char *value) @@ -463,10 +607,8 @@ int __system_property_update(prop_info *pi, const char *value, unsigned int len) int __system_property_add(const char *name, unsigned int namelen, const char *value, unsigned int valuelen) { - prop_area *pa; - prop_info *pa_info_array; - prop_info *pi; - size_t region; + prop_area *pa = __system_property_regions__[0]; + const prop_info *pi; if (namelen >= PROP_NAME_MAX) return -1; @@ -475,34 +617,12 @@ int __system_property_add(const char *name, unsigned int namelen, if (namelen < 1) return -1; - for (region = 0; region < PA_REGION_COUNT; region++) - { - int err = map_prop_region_rw(region); - if (err < 0) - return -1; - - pa = __system_property_regions__[region]; - - if (pa->count < PA_COUNT_MAX) - break; - } - - if (region == PA_REGION_COUNT) + pi = find_property(root_node(), name, namelen, value, valuelen, true); + if (!pi) return -1; - pa_info_array = (void*) (((char*) pa) + PA_INFO_START); - pi = pa_info_array + pa->count; - pi->serial = (valuelen << 24); - memcpy(pi->name, name, namelen + 1); - memcpy(pi->value, value, valuelen + 1); - - pa->toc[pa->count] = (namelen << 24) | (((unsigned) pi) - ((unsigned) pa)); - ANDROID_MEMBAR_FULL(); - - pa->count++; pa->serial++; __futex_wake(&pa->serial, INT32_MAX); - return 0; } @@ -521,3 +641,72 @@ unsigned int __system_property_wait_any(unsigned int serial) return pa->serial; } + +struct find_nth_cookie { + unsigned count; + unsigned n; + const prop_info *pi; +}; + +static void find_nth_fn(const prop_info *pi, void *ptr) +{ + struct find_nth_cookie *cookie = ptr; + + if (cookie->n == cookie->count) + cookie->pi = pi; + + cookie->count++; +} + +const prop_info *__system_property_find_nth(unsigned n) +{ + struct find_nth_cookie cookie; + int err; + + memset(&cookie, 0, sizeof(cookie)); + cookie.n = n; + + err = __system_property_foreach(find_nth_fn, &cookie); + if (err < 0) + return NULL; + + return cookie.pi; +} + +static int foreach_property(prop_off_t off, + void (*propfn)(const prop_info *pi, void *cookie), void *cookie) +{ + prop_bt *trie = to_prop_obj(off); + if (!trie) + return -1; + + if (trie->left) { + int err = foreach_property(trie->left, propfn, cookie); + if (err < 0) + return -1; + } + if (trie->prop) { + prop_info *info = to_prop_obj(trie->prop); + if (!info) + return -1; + propfn(info, cookie); + } + if (trie->children) { + int err = foreach_property(trie->children, propfn, cookie); + if (err < 0) + return -1; + } + if (trie->right) { + int err = foreach_property(trie->right, propfn, cookie); + if (err < 0) + return -1; + } + + return 0; +} + +int __system_property_foreach(void (*propfn)(const prop_info *pi, void *cookie), + void *cookie) +{ + return foreach_property(0, propfn, cookie); +} diff --git a/libc/include/sys/_system_properties.h b/libc/include/sys/_system_properties.h index 702a4b265..9c48e1b3e 100644 --- a/libc/include/sys/_system_properties.h +++ b/libc/include/sys/_system_properties.h @@ -37,7 +37,7 @@ typedef struct prop_msg prop_msg; #define PROP_AREA_MAGIC 0x504f5250 -#define PROP_AREA_VERSION 0x45434f76 +#define PROP_AREA_VERSION 0xfc6ed0ab #define PROP_SERVICE_NAME "property_service" #define PROP_FILENAME "/dev/__properties__" @@ -45,13 +45,8 @@ typedef struct prop_msg prop_msg; /* (4 header words + 28 toc words) = 128 bytes */ /* 128 bytes header and toc + 28 prop_infos @ 128 bytes = 3712 bytes */ -#define PA_COUNT_MAX 247 -#define PA_REGION_COUNT 16 -#define PA_INFO_START 1024 -#define PA_SIZE 32768 - -#define TOC_NAME_LEN(toc) ((toc) >> 24) -#define TOC_TO_INFO(area, toc) ((prop_info*) (((char*) area) + ((toc) & 0xFFFFFF))) +#define PA_REGION_COUNT 128 +#define PA_SIZE 4096 #define SERIAL_VALUE_LEN(serial) ((serial) >> 24) #define SERIAL_DIRTY(serial) ((serial) & 1) @@ -84,11 +79,6 @@ struct prop_msg ** 1. pi->serial = pi->serial | 1 ** 2. memcpy(pi->value, local_value, value_len) ** 3. pi->serial = (value_len << 24) | ((pi->serial + 1) & 0xffffff) -** -** Improvements: -** - maintain the toc sorted by pi->name to allow lookup -** by binary search -** */ #define PROP_PATH_RAMDISK_DEFAULT "/default.prop" From d5276422ca9f1f4d45e91c189a1655521e91962d Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Mon, 17 Jun 2013 12:37:09 -0700 Subject: [PATCH 37/77] bionic: add hierarchical properties test Deliberately put items several levels deep in the trie hierarchy to test the trie traversal Change-Id: I995a1cdd3b5e74162fb5d25bc0f65140bdf2f719 Signed-off-by: Greg Hackmann --- tests/system_properties_test.cpp | 78 ++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/tests/system_properties_test.cpp b/tests/system_properties_test.cpp index 50bdfdfe4..bafd2c3e1 100644 --- a/tests/system_properties_test.cpp +++ b/tests/system_properties_test.cpp @@ -202,6 +202,84 @@ TEST(properties, find_nth) { ASSERT_EQ((const prop_info *)NULL, __system_property_find_nth(247)); } +static void hierarchical_test_callback(const prop_info *pi, void *cookie) { + bool (*ok)[8][8] = static_cast(cookie); + + char name[PROP_NAME_MAX]; + char value[PROP_VALUE_MAX]; + + __system_property_read(pi, name, value); + + int name_i, name_j, name_k; + int value_i, value_j, value_k; + ASSERT_EQ(3, sscanf(name, "property_%d.%d.%d", &name_i, &name_j, &name_k)); + ASSERT_EQ(3, sscanf(value, "value_%d.%d.%d", &value_i, &value_j, &value_k)); + ASSERT_EQ(name_i, value_i); + ASSERT_GE(name_i, 0); + ASSERT_LT(name_i, 8); + ASSERT_EQ(name_j, value_j); + ASSERT_GE(name_j, 0); + ASSERT_LT(name_j, 8); + ASSERT_EQ(name_k, value_k); + ASSERT_GE(name_k, 0); + ASSERT_LT(name_k, 8); + + ok[name_i][name_j][name_k] = true; +} + +TEST(properties, fill_hierarchical) { + LocalPropertyTestState pa; + ASSERT_TRUE(pa.valid); + char prop_name[PROP_NAME_MAX]; + char prop_value[PROP_VALUE_MAX]; + char prop_value_ret[PROP_VALUE_MAX]; + int ret; + + for (int i = 0; i < 8; i++) { + for (int j = 0; j < 8; j++) { + for (int k = 0; k < 8; k++) { + ret = snprintf(prop_name, PROP_NAME_MAX - 1, "property_%d.%d.%d", i, j, k); + memset(prop_name + ret, 'a', PROP_NAME_MAX - 1 - ret); + ret = snprintf(prop_value, PROP_VALUE_MAX - 1, "value_%d.%d.%d", i, j, k); + memset(prop_value + ret, 'b', PROP_VALUE_MAX - 1 - ret); + prop_name[PROP_NAME_MAX - 1] = 0; + prop_value[PROP_VALUE_MAX - 1] = 0; + + ASSERT_EQ(0, __system_property_add(prop_name, PROP_NAME_MAX - 1, prop_value, PROP_VALUE_MAX - 1)); + } + } + } + + for (int i = 0; i < 8; i++) { + for (int j = 0; j < 8; j++) { + for (int k = 0; k < 8; k++) { + ret = snprintf(prop_name, PROP_NAME_MAX - 1, "property_%d.%d.%d", i, j, k); + memset(prop_name + ret, 'a', PROP_NAME_MAX - 1 - ret); + ret = snprintf(prop_value, PROP_VALUE_MAX - 1, "value_%d.%d.%d", i, j, k); + memset(prop_value + ret, 'b', PROP_VALUE_MAX - 1 - ret); + prop_name[PROP_NAME_MAX - 1] = 0; + prop_value[PROP_VALUE_MAX - 1] = 0; + memset(prop_value_ret, '\0', PROP_VALUE_MAX); + + ASSERT_EQ(PROP_VALUE_MAX - 1, __system_property_get(prop_name, prop_value_ret)); + ASSERT_EQ(0, memcmp(prop_value, prop_value_ret, PROP_VALUE_MAX)); + } + } + } + + bool ok[8][8][8]; + memset(ok, 0, sizeof(ok)); + __system_property_foreach(hierarchical_test_callback, ok); + + for (int i = 0; i < 8; i++) { + for (int j = 0; j < 8; j++) { + for (int k = 0; k < 8; k++) { + ASSERT_TRUE(ok[i][j][k]); + } + } + } +} + TEST(properties, errors) { LocalPropertyTestState pa; ASSERT_TRUE(pa.valid); From be30c7a78a1cf4adc8ec9bbdf17e85186fdb05b2 Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Wed, 19 Jun 2013 13:31:21 -0700 Subject: [PATCH 38/77] bionic: revert to a single (larger) property area d329697 is too complicated. Change the multiple property pages back to a single 128K property area that's mapped in entirely at initialization (the memory will not get allocated until the pages are touched). d329697 has other changes useful for testing (moving property area initialization inside bionic and adding __system_property_set_filename) so undo the change manually rather than with git revert. Change-Id: Icd137669a4f8bc248e9dd2c1e8cc54e9193c9a6d Signed-off-by: Greg Hackmann --- libc/bionic/system_properties.c | 91 ++++++++------------------- libc/include/sys/_system_properties.h | 6 +- tests/property_benchmark.cpp | 14 ++--- tests/system_properties_test.cpp | 14 ++--- 4 files changed, 36 insertions(+), 89 deletions(-) diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index 126fea56a..d4054d245 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -109,7 +109,7 @@ typedef struct prop_bt prop_bt; static const char property_service_socket[] = "/dev/socket/" PROP_SERVICE_NAME; static char property_filename[PATH_MAX] = PROP_FILENAME; -prop_area *__system_property_regions__[PA_REGION_COUNT] = { NULL, }; +prop_area *__system_property_area__ = NULL; const size_t PA_DATA_SIZE = PA_SIZE - sizeof(prop_area); @@ -124,15 +124,10 @@ static int get_fd_from_env(void) return atoi(env); } -static int map_prop_region_rw(size_t region) +static int map_prop_area_rw() { prop_area *pa; int fd; - size_t offset = region * PA_SIZE; - - if (__system_property_regions__[region]) { - return 0; - } /* dev is a tmpfs that we can use to carve a shared workspace * out of, so let's do that... @@ -148,24 +143,21 @@ static int map_prop_region_rw(size_t region) return -1; } - if (ftruncate(fd, offset + PA_SIZE) < 0) + if (ftruncate(fd, PA_SIZE) < 0) goto out; - pa = mmap(NULL, PA_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, offset); + pa = mmap(NULL, PA_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if(pa == MAP_FAILED) goto out; memset(pa, 0, PA_SIZE); pa->magic = PROP_AREA_MAGIC; pa->version = PROP_AREA_VERSION; - - if (!region) { - /* reserve root node */ - pa->bytes_used += sizeof(prop_bt); - } + /* reserve root node */ + pa->bytes_used = sizeof(prop_bt); /* plug into the lib property services */ - __system_property_regions__[region] = pa; + __system_property_area__ = pa; close(fd); return 0; @@ -187,20 +179,14 @@ int __system_property_set_filename(const char *filename) int __system_property_area_init() { - return map_prop_region_rw(0); + return map_prop_area_rw(); } -static int map_prop_region(size_t region) +static int map_prop_area() { bool fromFile = true; - bool swapped; - size_t offset = region * PA_SIZE; int result = -1; - if(__system_property_regions__[region]) { - return 0; - } - int fd = open(property_filename, O_RDONLY | O_NOFOLLOW); if ((fd < 0) && (errno == ENOENT)) { @@ -229,11 +215,11 @@ static int map_prop_region(size_t region) if ((fd_stat.st_uid != 0) || (fd_stat.st_gid != 0) || ((fd_stat.st_mode & (S_IWGRP | S_IWOTH)) != 0) - || (fd_stat.st_size < offset + PA_SIZE) ) { + || (fd_stat.st_size < PA_SIZE) ) { goto cleanup; } - prop_area *pa = mmap(NULL, PA_SIZE, PROT_READ, MAP_SHARED, fd, offset); + prop_area *pa = mmap(NULL, PA_SIZE, PROT_READ, MAP_SHARED, fd, 0); if (pa == MAP_FAILED) { goto cleanup; @@ -245,16 +231,8 @@ static int map_prop_region(size_t region) } result = 0; - swapped = __sync_bool_compare_and_swap(&__system_property_regions__[region], - NULL, pa); - if (!swapped) { - /** - * In the event of a race either mapping is equally good, so - * the thread that lost can just throw its mapping away and proceed as - * normal. - */ - munmap(pa, PA_SIZE); - } + + __system_property_area__ = pa; cleanup: if (fromFile) { @@ -266,33 +244,20 @@ cleanup: int __system_properties_init() { - return map_prop_region(0); + return map_prop_area(); } static void *new_prop_obj(size_t size, prop_off_t *off) { - prop_area *pa; - size_t i, idx; + prop_area *pa = __system_property_area__; size = ALIGN(size, sizeof(uint32_t)); - for (i = 0; i < PA_REGION_COUNT; i++) { - int err = map_prop_region_rw(i); - if (err < 0) { - return NULL; - } - - pa = __system_property_regions__[i]; - if (pa->bytes_used + size <= PA_DATA_SIZE) - break; - } - - if (i == PA_REGION_COUNT) + if (pa->bytes_used + size > PA_DATA_SIZE) return NULL; - idx = pa->bytes_used; - *off = idx + i * PA_DATA_SIZE; - pa->bytes_used += size; - return pa->data + idx; + *off = pa->bytes_used; + __system_property_area__->bytes_used += size; + return __system_property_area__->data + *off; } static prop_bt *new_prop_bt(const char *name, uint8_t namelen, prop_off_t *off) @@ -330,16 +295,10 @@ static prop_info *new_prop_info(const char *name, uint8_t namelen, static void *to_prop_obj(prop_off_t off) { - size_t region = off / PA_DATA_SIZE; - size_t idx = off % PA_DATA_SIZE; - - if (region > PA_REGION_COUNT) + if (off > PA_DATA_SIZE) return NULL; - if (map_prop_region(region) < 0) - return NULL; - - return __system_property_regions__[region]->data + idx; + return __system_property_area__->data + off; } static prop_bt *root_node() @@ -570,7 +529,7 @@ int __system_property_wait(const prop_info *pi) { unsigned n; if(pi == 0) { - prop_area *pa = __system_property_regions__[0]; + prop_area *pa = __system_property_area__; n = pa->serial; do { __futex_wait(&pa->serial, n, 0); @@ -586,7 +545,7 @@ int __system_property_wait(const prop_info *pi) int __system_property_update(prop_info *pi, const char *value, unsigned int len) { - prop_area *pa = __system_property_regions__[0]; + prop_area *pa = __system_property_area__; if (len >= PROP_VALUE_MAX) return -1; @@ -607,7 +566,7 @@ int __system_property_update(prop_info *pi, const char *value, unsigned int len) int __system_property_add(const char *name, unsigned int namelen, const char *value, unsigned int valuelen) { - prop_area *pa = __system_property_regions__[0]; + prop_area *pa = __system_property_area__; const prop_info *pi; if (namelen >= PROP_NAME_MAX) @@ -633,7 +592,7 @@ unsigned int __system_property_serial(const prop_info *pi) unsigned int __system_property_wait_any(unsigned int serial) { - prop_area *pa = __system_property_regions__[0]; + prop_area *pa = __system_property_area__; do { __futex_wait(&pa->serial, serial, 0); diff --git a/libc/include/sys/_system_properties.h b/libc/include/sys/_system_properties.h index 9c48e1b3e..92e35e124 100644 --- a/libc/include/sys/_system_properties.h +++ b/libc/include/sys/_system_properties.h @@ -42,11 +42,7 @@ typedef struct prop_msg prop_msg; #define PROP_SERVICE_NAME "property_service" #define PROP_FILENAME "/dev/__properties__" -/* (4 header words + 28 toc words) = 128 bytes */ -/* 128 bytes header and toc + 28 prop_infos @ 128 bytes = 3712 bytes */ - -#define PA_REGION_COUNT 128 -#define PA_SIZE 4096 +#define PA_SIZE (128 * 1024) #define SERIAL_VALUE_LEN(serial) ((serial) >> 24) #define SERIAL_DIRTY(serial) ((serial) & 1) diff --git a/tests/property_benchmark.cpp b/tests/property_benchmark.cpp index 7266bd08a..d10be915e 100644 --- a/tests/property_benchmark.cpp +++ b/tests/property_benchmark.cpp @@ -23,7 +23,7 @@ #include #include -extern void *__system_property_regions__[PA_REGION_COUNT]; +extern void *__system_property_area__; #define TEST_NUM_PROPS \ Arg(1)->Arg(4)->Arg(16)->Arg(64)->Arg(128)->Arg(256)->Arg(512)->Arg(1024) @@ -39,10 +39,8 @@ struct LocalPropertyTestState { return; } - for (size_t i = 0; i < PA_REGION_COUNT; i++) { - old_pa[i] = __system_property_regions__[i]; - __system_property_regions__[i] = NULL; - } + old_pa = __system_property_area__; + __system_property_area__ = NULL; pa_dirname = dirname; pa_filename = pa_dirname + "/__properties__"; @@ -79,9 +77,7 @@ struct LocalPropertyTestState { if (!valid) return; - for (size_t i = 0; i < PA_REGION_COUNT; i++) { - __system_property_regions__[i] = old_pa[i]; - } + __system_property_area__ = old_pa; __system_property_set_filename(PROP_FILENAME); unlink(pa_filename.c_str()); @@ -107,7 +103,7 @@ public: private: std::string pa_dirname; std::string pa_filename; - void *old_pa[PA_REGION_COUNT]; + void *old_pa; }; static void BM_property_get(int iters, int nprops) diff --git a/tests/system_properties_test.cpp b/tests/system_properties_test.cpp index bafd2c3e1..b9256c664 100644 --- a/tests/system_properties_test.cpp +++ b/tests/system_properties_test.cpp @@ -24,7 +24,7 @@ #define _REALLY_INCLUDE_SYS__SYSTEM_PROPERTIES_H_ #include -extern void *__system_property_regions__[PA_REGION_COUNT]; +extern void *__system_property_area__; struct LocalPropertyTestState { LocalPropertyTestState() : valid(false) { @@ -35,10 +35,8 @@ struct LocalPropertyTestState { return; } - for (size_t i = 0; i < PA_REGION_COUNT; i++) { - old_pa[i] = __system_property_regions__[i]; - __system_property_regions__[i] = NULL; - } + old_pa = __system_property_area__; + __system_property_area__ = NULL; pa_dirname = dirname; pa_filename = pa_dirname + "/__properties__"; @@ -52,9 +50,7 @@ struct LocalPropertyTestState { if (!valid) return; - for (size_t i = 0; i < PA_REGION_COUNT; i++) { - __system_property_regions__[i] = old_pa[i]; - } + __system_property_area__ = old_pa; __system_property_set_filename(PROP_FILENAME); unlink(pa_filename.c_str()); @@ -65,7 +61,7 @@ public: private: std::string pa_dirname; std::string pa_filename; - void *old_pa[PA_REGION_COUNT]; + void *old_pa; }; TEST(properties, add) { From fc761f8cc0dca78ac4ace20f701554704a6c14d5 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Sun, 16 Jun 2013 10:19:16 -0700 Subject: [PATCH 39/77] bionic: prevent root processes from calling __system_property_add If a root process other than init calls __system_property_add, which it should never do, it will break the design assumption that there is only one mutator. Pass O_EXCL to open() in map_prop_region_rw to ensure that only one process ever has the property pages open for write. Change-Id: I8233bfe0beaa40a5003ad53b98c661536b15f6b7 --- libc/bionic/system_properties.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index d4054d245..f9671c636 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -128,11 +128,13 @@ static int map_prop_area_rw() { prop_area *pa; int fd; + int ret; /* dev is a tmpfs that we can use to carve a shared workspace * out of, so let's do that... */ - fd = open(property_filename, O_RDWR | O_CREAT | O_NOFOLLOW, 0644); + fd = open(property_filename, O_RDWR | O_CREAT | O_NOFOLLOW | O_CLOEXEC | + O_EXCL, 0444); if (fd < 0) { if (errno == EACCES) { /* for consistency with the case where the process has already @@ -143,6 +145,10 @@ static int map_prop_area_rw() return -1; } + ret = fcntl(fd, F_SETFD, FD_CLOEXEC); + if (ret < 0) + goto out; + if (ftruncate(fd, PA_SIZE) < 0) goto out; @@ -186,8 +192,16 @@ static int map_prop_area() { bool fromFile = true; int result = -1; + int fd; + int ret; - int fd = open(property_filename, O_RDONLY | O_NOFOLLOW); + fd = open(property_filename, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (fd >= 0) { + /* For old kernels that don't support O_CLOEXEC */ + ret = fcntl(fd, F_SETFD, FD_CLOEXEC); + if (ret < 0) + goto cleanup; + } if ((fd < 0) && (errno == ENOENT)) { /* From 4fe82929eb19b35cbd144aace00f71fa533e7bce Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Fri, 21 Jun 2013 13:02:38 -0700 Subject: [PATCH 40/77] bionic: store property names as variable-length strings Names are immutable, so the fixed-sized arrays can be replaced with variable-length ones to save memory (especially on internal tree nodes). Change-Id: Iddf7856fba579b97f355e9ad4b3663a78767b96d Signed-off-by: Greg Hackmann --- libc/bionic/system_properties.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index f9671c636..481e6ae45 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -66,9 +66,9 @@ struct prop_area { typedef struct prop_area prop_area; struct prop_info { - char name[PROP_NAME_MAX]; unsigned volatile serial; char value[PROP_VALUE_MAX]; + char name[0]; }; typedef struct prop_info prop_info; @@ -92,7 +92,6 @@ typedef struct prop_info prop_info; typedef volatile uint32_t prop_off_t; struct prop_bt { - char name[PROP_NAME_MAX]; uint8_t namelen; uint8_t reserved[3]; @@ -102,6 +101,8 @@ struct prop_bt { prop_off_t right; prop_off_t children; + + char name[0]; }; typedef struct prop_bt prop_bt; @@ -277,7 +278,7 @@ static void *new_prop_obj(size_t size, prop_off_t *off) static prop_bt *new_prop_bt(const char *name, uint8_t namelen, prop_off_t *off) { prop_off_t off_tmp; - prop_bt *bt = new_prop_obj(sizeof(prop_bt), &off_tmp); + prop_bt *bt = new_prop_obj(sizeof(prop_bt) + namelen + 1, &off_tmp); if (bt) { memcpy(bt->name, name, namelen); bt->name[namelen] = '\0'; @@ -293,7 +294,7 @@ static prop_info *new_prop_info(const char *name, uint8_t namelen, const char *value, uint8_t valuelen, prop_off_t *off) { prop_off_t off_tmp; - prop_info *info = new_prop_obj(sizeof(prop_info), &off_tmp); + prop_info *info = new_prop_obj(sizeof(prop_info) + namelen + 1, &off_tmp); if (info) { memcpy(info->name, name, namelen); info->name[namelen] = '\0'; From 285b42a04cbe8e627a75c9bfb3c7cb3f7b539267 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Mon, 24 Jun 2013 18:42:21 -0700 Subject: [PATCH 41/77] bionic: use the size of the file to determine property area size On the reader size, don't assume that the property size is PA_SIZE, read it from the size of the file. Allows init to use a different property size without recompiling statically linked executables. Change-Id: I87fb0bf40c4724e3759a583fd9ea1f49492bc958 --- libc/bionic/system_properties.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index 481e6ae45..0d421956e 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -112,7 +112,8 @@ static char property_filename[PATH_MAX] = PROP_FILENAME; prop_area *__system_property_area__ = NULL; -const size_t PA_DATA_SIZE = PA_SIZE - sizeof(prop_area); +size_t pa_data_size; +size_t pa_size; static int get_fd_from_env(void) { @@ -153,11 +154,14 @@ static int map_prop_area_rw() if (ftruncate(fd, PA_SIZE) < 0) goto out; - pa = mmap(NULL, PA_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + pa_size = PA_SIZE; + pa_data_size = pa_size - sizeof(prop_area); + + pa = mmap(NULL, pa_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if(pa == MAP_FAILED) goto out; - memset(pa, 0, PA_SIZE); + memset(pa, 0, pa_size); pa->magic = PROP_AREA_MAGIC; pa->version = PROP_AREA_VERSION; /* reserve root node */ @@ -230,18 +234,20 @@ static int map_prop_area() if ((fd_stat.st_uid != 0) || (fd_stat.st_gid != 0) || ((fd_stat.st_mode & (S_IWGRP | S_IWOTH)) != 0) - || (fd_stat.st_size < PA_SIZE) ) { + || (fd_stat.st_size < sizeof(prop_area)) ) { goto cleanup; } - prop_area *pa = mmap(NULL, PA_SIZE, PROT_READ, MAP_SHARED, fd, 0); + pa_size = fd_stat.st_size; + pa_data_size = pa_size - sizeof(prop_area); + prop_area *pa = mmap(NULL, pa_size, PROT_READ, MAP_SHARED, fd, 0); if (pa == MAP_FAILED) { goto cleanup; } if((pa->magic != PROP_AREA_MAGIC) || (pa->version != PROP_AREA_VERSION)) { - munmap(pa, PA_SIZE); + munmap(pa, pa_size); goto cleanup; } @@ -267,7 +273,7 @@ static void *new_prop_obj(size_t size, prop_off_t *off) prop_area *pa = __system_property_area__; size = ALIGN(size, sizeof(uint32_t)); - if (pa->bytes_used + size > PA_DATA_SIZE) + if (pa->bytes_used + size > pa_data_size) return NULL; *off = pa->bytes_used; @@ -310,7 +316,7 @@ static prop_info *new_prop_info(const char *name, uint8_t namelen, static void *to_prop_obj(prop_off_t off) { - if (off > PA_DATA_SIZE) + if (off > pa_data_size) return NULL; return __system_property_area__->data + off; From ad76c85b9ca587084089d086f954158bc0eae905 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Mon, 24 Jun 2013 18:36:39 -0700 Subject: [PATCH 42/77] bionic: add compatibility mode for properties Allow a new bionic to work with an old init property area by supporting the old format. Change-Id: I9268214b2f8930e75e3b5c26afe94fa92068bf0b --- libc/Android.mk | 1 + libc/bionic/system_properties.c | 19 +++- libc/bionic/system_properties_compat.c | 131 +++++++++++++++++++++++++ libc/include/sys/_system_properties.h | 10 ++ 4 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 libc/bionic/system_properties_compat.c diff --git a/libc/Android.mk b/libc/Android.mk index 11ae09b0f..7b2299969 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -150,6 +150,7 @@ libc_common_src_files := \ bionic/strntoumax.c \ bionic/strtotimeval.c \ bionic/system_properties.c \ + bionic/system_properties_compat.c \ bionic/tcgetpgrp.c \ bionic/tcsetpgrp.c \ bionic/thread_atexit.c \ diff --git a/libc/bionic/system_properties.c b/libc/bionic/system_properties.c index 0d421956e..4c2e5a21f 100644 --- a/libc/bionic/system_properties.c +++ b/libc/bionic/system_properties.c @@ -109,6 +109,7 @@ typedef struct prop_bt prop_bt; static const char property_service_socket[] = "/dev/socket/" PROP_SERVICE_NAME; static char property_filename[PATH_MAX] = PROP_FILENAME; +static bool compat_mode = false; prop_area *__system_property_area__ = NULL; @@ -156,6 +157,7 @@ static int map_prop_area_rw() pa_size = PA_SIZE; pa_data_size = pa_size - sizeof(prop_area); + compat_mode = false; pa = mmap(NULL, pa_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if(pa == MAP_FAILED) @@ -246,11 +248,16 @@ static int map_prop_area() goto cleanup; } - if((pa->magic != PROP_AREA_MAGIC) || (pa->version != PROP_AREA_VERSION)) { + if((pa->magic != PROP_AREA_MAGIC) || (pa->version != PROP_AREA_VERSION && + pa->version != PROP_AREA_VERSION_COMPAT)) { munmap(pa, pa_size); goto cleanup; } + if (pa->version == PROP_AREA_VERSION_COMPAT) { + compat_mode = true; + } + result = 0; __system_property_area__ = pa; @@ -425,6 +432,9 @@ static const prop_info *find_property(prop_bt *trie, const char *name, const prop_info *__system_property_find(const char *name) { + if (__predict_false(compat_mode)) { + return __system_property_find_compat(name); + } return find_property(root_node(), name, strlen(name), NULL, 0, false); } @@ -432,6 +442,10 @@ int __system_property_read(const prop_info *pi, char *name, char *value) { unsigned serial, len; + if (__predict_false(compat_mode)) { + return __system_property_read_compat(pi, name, value); + } + for(;;) { serial = pi->serial; while(SERIAL_DIRTY(serial)) { @@ -688,5 +702,8 @@ static int foreach_property(prop_off_t off, int __system_property_foreach(void (*propfn)(const prop_info *pi, void *cookie), void *cookie) { + if (__predict_false(compat_mode)) { + return __system_property_foreach_compat(propfn, cookie); + } return foreach_property(0, propfn, cookie); } diff --git a/libc/bionic/system_properties_compat.c b/libc/bionic/system_properties_compat.c new file mode 100644 index 000000000..6dbc4ccea --- /dev/null +++ b/libc/bionic/system_properties_compat.c @@ -0,0 +1,131 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * This file is only used to provide backwards compatibility to property areas + * created by old versions of init, which occurs when an ota runs. The updater + * binary is compiled statically against the newest bionic, but the recovery + * ramdisk may be using an old version of init. This can all be removed once + * OTAs from pre-K versions are no longer supported. + */ + +#include +#include + +#define _REALLY_INCLUDE_SYS__SYSTEM_PROPERTIES_H_ +#include + +#define TOC_NAME_LEN(toc) ((toc) >> 24) +#define TOC_TO_INFO(area, toc) ((prop_info_compat*) (((char*) area) + ((toc) & 0xFFFFFF))) + +struct prop_area_compat { + unsigned volatile count; + unsigned volatile serial; + unsigned magic; + unsigned version; + unsigned toc[1]; +}; + +typedef struct prop_area_compat prop_area_compat; + +struct prop_area; +typedef struct prop_area prop_area; + +struct prop_info_compat { + char name[PROP_NAME_MAX]; + unsigned volatile serial; + char value[PROP_VALUE_MAX]; +}; + +typedef struct prop_info_compat prop_info_compat; + +extern prop_area *__system_property_area__; + +const prop_info *__system_property_find_compat(const char *name) +{ + prop_area_compat *pa = (prop_area_compat *)__system_property_area__; + unsigned count = pa->count; + unsigned *toc = pa->toc; + unsigned len = strlen(name); + prop_info_compat *pi; + + if (len >= PROP_NAME_MAX) + return 0; + if (len < 1) + return 0; + + while(count--) { + unsigned entry = *toc++; + if(TOC_NAME_LEN(entry) != len) continue; + + pi = TOC_TO_INFO(pa, entry); + if(memcmp(name, pi->name, len)) continue; + + return (const prop_info *)pi; + } + + return 0; +} + +int __system_property_read_compat(const prop_info *_pi, char *name, char *value) +{ + unsigned serial, len; + const prop_info_compat *pi = (const prop_info_compat *)_pi; + + for(;;) { + serial = pi->serial; + while(SERIAL_DIRTY(serial)) { + __futex_wait((volatile void *)&pi->serial, serial, 0); + serial = pi->serial; + } + len = SERIAL_VALUE_LEN(serial); + memcpy(value, pi->value, len + 1); + if(serial == pi->serial) { + if(name != 0) { + strcpy(name, pi->name); + } + return len; + } + } +} + +int __system_property_foreach_compat( + void (*propfn)(const prop_info *pi, void *cookie), + void *cookie) +{ + prop_area_compat *pa = (prop_area_compat *)__system_property_area__; + unsigned i; + + for (i = 0; i < pa->count; i++) { + unsigned entry = pa->toc[i]; + prop_info_compat *pi = TOC_TO_INFO(pa, entry); + propfn((const prop_info *)pi, cookie); + } + + return 0; +} diff --git a/libc/include/sys/_system_properties.h b/libc/include/sys/_system_properties.h index 92e35e124..5eee7f097 100644 --- a/libc/include/sys/_system_properties.h +++ b/libc/include/sys/_system_properties.h @@ -38,6 +38,7 @@ typedef struct prop_msg prop_msg; #define PROP_AREA_MAGIC 0x504f5250 #define PROP_AREA_VERSION 0xfc6ed0ab +#define PROP_AREA_VERSION_COMPAT 0x45434f76 #define PROP_SERVICE_NAME "property_service" #define PROP_FILENAME "/dev/__properties__" @@ -129,6 +130,15 @@ unsigned int __system_property_serial(const prop_info *pi); ** successive call. */ unsigned int __system_property_wait_any(unsigned int serial); +/* Compatibility functions to support using an old init with a new libc, + ** mostly for the OTA updater binary. These can be deleted once OTAs from + ** a pre-K release no longer needed to be supported. */ +const prop_info *__system_property_find_compat(const char *name); +int __system_property_read_compat(const prop_info *pi, char *name, char *value); +int __system_property_foreach_compat( + void (*propfn)(const prop_info *pi, void *cookie), + void *cookie); + __END_DECLS #endif From 553a5ec407ddc922ffca7f542e280b6150396cb9 Mon Sep 17 00:00:00 2001 From: Nick Kralevich Date: Tue, 2 Jul 2013 12:28:03 -0700 Subject: [PATCH 43/77] mmap: Don't call madvise if mmap fails Fix improper check of return value from __mmap2. On a failed mmap(), it returns MAP_FAILED, not 0. This ended up clobbering errno when madvise subsequently failed. Change-Id: I364fb2f158fe258c55a73e552195384b2c11c193 --- libc/bionic/mmap.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libc/bionic/mmap.c b/libc/bionic/mmap.c index e097086cb..36aee85f2 100644 --- a/libc/bionic/mmap.c +++ b/libc/bionic/mmap.c @@ -43,8 +43,9 @@ void* mmap(void *addr, size_t size, int prot, int flags, int fd, long offset) ret = __mmap2(addr, size, prot, flags, fd, (size_t)offset >> MMAP2_SHIFT); - if (ret && (flags & (MAP_PRIVATE | MAP_ANONYMOUS))) - madvise(ret, size, MADV_MERGEABLE); + if ((ret != MAP_FAILED) && (flags & (MAP_PRIVATE | MAP_ANONYMOUS))) { + madvise(ret, size, MADV_MERGEABLE); + } return ret; } From c39214e0ac49b9f7de6fd9989331145c6e1cd584 Mon Sep 17 00:00:00 2001 From: Chad Brubaker Date: Thu, 20 Jun 2013 10:36:56 -0700 Subject: [PATCH 44/77] Add marking of DNS sockets for mark based routing Adds an extra mark parameter to android_getaddrinfoforiface, android_gethostbyaddrforiface and android_gethostbynameforiface that if set will cause all packets sent by DNS requests to have that mark Change-Id: I6f72390e4ce5bfc3cc73183f9b2fb3705a11685f --- libc/include/netdb.h | 8 ++++---- libc/netbsd/gethnamaddr.c | 32 ++++++++++++++++++-------------- libc/netbsd/net/getaddrinfo.c | 15 +++++++++------ libc/netbsd/net/getnameinfo.c | 19 ++++++++++--------- libc/netbsd/resolv/res_init.c | 7 +++++++ libc/netbsd/resolv/res_send.c | 24 ++++++++++++++++++++++-- libc/private/resolv_private.h | 2 ++ 7 files changed, 72 insertions(+), 35 deletions(-) diff --git a/libc/include/netdb.h b/libc/include/netdb.h index 3ea512c75..62a7a3cc8 100644 --- a/libc/include/netdb.h +++ b/libc/include/netdb.h @@ -207,13 +207,13 @@ void endprotoent(void); void endservent(void); void freehostent(struct hostent *); struct hostent *gethostbyaddr(const void *, socklen_t, int); -struct hostent *android_gethostbyaddrforiface(const void *, socklen_t, int, const char*); +struct hostent *android_gethostbyaddrforiface(const void *, socklen_t, int, const char*, int); int gethostbyaddr_r(const void *, int, int, struct hostent *, char *, size_t, struct hostent **, int *); struct hostent *gethostbyname(const char *); int gethostbyname_r(const char *, struct hostent *, char *, size_t, struct hostent **, int *); struct hostent *gethostbyname2(const char *, int); int gethostbyname2_r(const char *, int, struct hostent *, char *, size_t, struct hostent **, int *); -struct hostent *android_gethostbynameforiface(const char *, int, const char *); +struct hostent *android_gethostbynameforiface(const char *, int, const char *, int); struct hostent *gethostent(void); int gethostent_r(struct hostent *, char *, size_t, struct hostent **, int *); struct hostent *getipnodebyaddr(const void *, size_t, int, int *); @@ -241,9 +241,9 @@ void sethostent(int); void setnetent(int); void setprotoent(int); int getaddrinfo(const char *, const char *, const struct addrinfo *, struct addrinfo **); -int android_getaddrinfoforiface(const char *, const char *, const struct addrinfo *, const char *, struct addrinfo **); +int android_getaddrinfoforiface(const char *, const char *, const struct addrinfo *, const char *, int, struct addrinfo **); int getnameinfo(const struct sockaddr *, socklen_t, char *, size_t, char *, size_t, int); -int android_getnameinfoforiface(const struct sockaddr *, socklen_t, char *, size_t, char *, size_t, int, const char *); +int android_getnameinfoforiface(const struct sockaddr *, socklen_t, char *, size_t, char *, size_t, int, const char *, int); void freeaddrinfo(struct addrinfo *); const char *gai_strerror(int); void setnetgrent(const char *); diff --git a/libc/netbsd/gethnamaddr.c b/libc/netbsd/gethnamaddr.c index ee5052e4c..5b2f98750 100644 --- a/libc/netbsd/gethnamaddr.c +++ b/libc/netbsd/gethnamaddr.c @@ -126,7 +126,7 @@ static struct hostent *_gethtbyname2(const char *, int); static int _dns_gethtbyaddr(void *, void *, va_list); static int _dns_gethtbyname(void *, void *, va_list); -static struct hostent *gethostbyname_internal(const char *, int, res_state, const char *); +static struct hostent *gethostbyname_internal(const char *, int, res_state, const char *, int); static const ns_src default_dns_files[] = { { NSSRC_FILES, NS_SUCCESS }, @@ -497,13 +497,13 @@ gethostbyname(const char *name) /* try IPv6 first - if that fails do IPv4 */ if (res->options & RES_USE_INET6) { - hp = gethostbyname_internal(name, AF_INET6, res, NULL); + hp = gethostbyname_internal(name, AF_INET6, res, NULL, 0); if (hp) { __res_put_state(res); return hp; } } - hp = gethostbyname_internal(name, AF_INET, res, NULL); + hp = gethostbyname_internal(name, AF_INET, res, NULL, 0); __res_put_state(res); return hp; } @@ -511,18 +511,18 @@ gethostbyname(const char *name) struct hostent * gethostbyname2(const char *name, int af) { - return android_gethostbynameforiface(name, af, NULL); + return android_gethostbynameforiface(name, af, NULL, 0); } struct hostent * -android_gethostbynameforiface(const char *name, int af, const char *iface) +android_gethostbynameforiface(const char *name, int af, const char *iface, int mark) { struct hostent *hp; res_state res = __res_get_state(); if (res == NULL) return NULL; - hp = gethostbyname_internal(name, af, res, iface); + hp = gethostbyname_internal(name, af, res, iface, mark); __res_put_state(res); return hp; } @@ -741,7 +741,7 @@ gethostbyname_internal_real(const char *name, int af, res_state res) // very similar in proxy-ness to android_getaddrinfo_proxy static struct hostent * -gethostbyname_internal(const char *name, int af, res_state res, const char *iface) +gethostbyname_internal(const char *name, int af, res_state res, const char *iface, int mark) { const char *cache_mode = getenv("ANDROID_DNS_MODE"); FILE* proxy = NULL; @@ -749,6 +749,7 @@ gethostbyname_internal(const char *name, int af, res_state res, const char *ifac if (cache_mode != NULL && strcmp(cache_mode, "local") == 0) { res_setiface(res, iface); + res_setmark(res, mark); return gethostbyname_internal_real(name, af, res); } @@ -780,7 +781,7 @@ exit: struct hostent * android_gethostbyaddrforiface_proxy(const void *addr, - socklen_t len, int af, const char* iface) + socklen_t len, int af, const char* iface, int mark) { struct hostent *result = NULL; FILE* proxy = android_open_proxy(); @@ -810,7 +811,7 @@ exit: struct hostent * android_gethostbyaddrforiface_real(const void *addr, - socklen_t len, int af, const char* iface) + socklen_t len, int af, const char* iface, int mark) { const u_char *uaddr = (const u_char *)addr; socklen_t size; @@ -858,28 +859,28 @@ android_gethostbyaddrforiface_real(const void *addr, hp = NULL; h_errno = NETDB_INTERNAL; if (nsdispatch(&hp, dtab, NSDB_HOSTS, "gethostbyaddr", - default_dns_files, uaddr, len, af, iface) != NS_SUCCESS) + default_dns_files, uaddr, len, af, iface, mark) != NS_SUCCESS) return NULL; h_errno = NETDB_SUCCESS; return hp; } struct hostent * -android_gethostbyaddrforiface(const void *addr, socklen_t len, int af, const char* iface) +android_gethostbyaddrforiface(const void *addr, socklen_t len, int af, const char* iface, int mark) { const char *cache_mode = getenv("ANDROID_DNS_MODE"); if (cache_mode == NULL || strcmp(cache_mode, "local") != 0) { - return android_gethostbyaddrforiface_proxy(addr, len, af, iface); + return android_gethostbyaddrforiface_proxy(addr, len, af, iface, mark); } else { - return android_gethostbyaddrforiface_real(addr,len, af,iface); + return android_gethostbyaddrforiface_real(addr,len, af, iface, mark); } } struct hostent * gethostbyaddr(const void *addr, socklen_t len, int af) { - return android_gethostbyaddrforiface(addr, len, af, NULL); + return android_gethostbyaddrforiface(addr, len, af, NULL, 0); } @@ -1315,6 +1316,7 @@ _dns_gethtbyaddr(void *rv, void *cb_data, va_list ap) int len, af, advance; res_state res; const char* iface; + int mark; res_static rs = __res_get_static(); assert(rv != NULL); @@ -1323,6 +1325,7 @@ _dns_gethtbyaddr(void *rv, void *cb_data, va_list ap) len = va_arg(ap, int); af = va_arg(ap, int); iface = va_arg(ap, char *); + mark = va_arg(ap, int); switch (af) { case AF_INET: @@ -1365,6 +1368,7 @@ _dns_gethtbyaddr(void *rv, void *cb_data, va_list ap) return NS_NOTFOUND; } res_setiface(res, iface); + res_setmark(res, mark); n = res_nquery(res, qbuf, C_IN, T_PTR, buf->buf, sizeof(buf->buf)); if (n < 0) { free(buf); diff --git a/libc/netbsd/net/getaddrinfo.c b/libc/netbsd/net/getaddrinfo.c index 401bc6efc..0a5e436ba 100644 --- a/libc/netbsd/net/getaddrinfo.c +++ b/libc/netbsd/net/getaddrinfo.c @@ -214,7 +214,7 @@ struct res_target { static int str2number(const char *); static int explore_fqdn(const struct addrinfo *, const char *, - const char *, struct addrinfo **, const char *iface); + const char *, struct addrinfo **, const char *iface, int mark); static int explore_null(const struct addrinfo *, const char *, struct addrinfo **); static int explore_numeric(const struct addrinfo *, const char *, @@ -577,12 +577,12 @@ int getaddrinfo(const char *hostname, const char *servname, const struct addrinfo *hints, struct addrinfo **res) { - return android_getaddrinfoforiface(hostname, servname, hints, NULL, res); + return android_getaddrinfoforiface(hostname, servname, hints, NULL, 0, res); } int android_getaddrinfoforiface(const char *hostname, const char *servname, - const struct addrinfo *hints, const char *iface, struct addrinfo **res) + const struct addrinfo *hints, const char *iface, int mark, struct addrinfo **res) { struct addrinfo sentinel; struct addrinfo *cur; @@ -761,7 +761,7 @@ android_getaddrinfoforiface(const char *hostname, const char *servname, pai->ai_protocol = ex->e_protocol; error = explore_fqdn(pai, hostname, servname, - &cur->ai_next, iface); + &cur->ai_next, iface, mark); while (cur && cur->ai_next) cur = cur->ai_next; @@ -794,7 +794,7 @@ android_getaddrinfoforiface(const char *hostname, const char *servname, */ static int explore_fqdn(const struct addrinfo *pai, const char *hostname, - const char *servname, struct addrinfo **res, const char *iface) + const char *servname, struct addrinfo **res, const char *iface, int mark) { struct addrinfo *result; struct addrinfo *cur; @@ -820,7 +820,7 @@ explore_fqdn(const struct addrinfo *pai, const char *hostname, return 0; switch (nsdispatch(&result, dtab, NSDB_HOSTS, "getaddrinfo", - default_dns_files, hostname, pai, iface)) { + default_dns_files, hostname, pai, iface, mark)) { case NS_TRYAGAIN: error = EAI_AGAIN; goto free; @@ -1889,10 +1889,12 @@ _dns_getaddrinfo(void *rv, void *cb_data, va_list ap) struct res_target q, q2; res_state res; const char* iface; + int mark; name = va_arg(ap, char *); pai = va_arg(ap, const struct addrinfo *); iface = va_arg(ap, char *); + mark = va_arg(ap, int); //fprintf(stderr, "_dns_getaddrinfo() name = '%s'\n", name); memset(&q, 0, sizeof(q)); @@ -1980,6 +1982,7 @@ _dns_getaddrinfo(void *rv, void *cb_data, va_list ap) * and have a cache hit that would be wasted, so we do the rest there on miss */ res_setiface(res, iface); + res_setmark(res, mark); if (res_searchN(name, &q, res) < 0) { __res_put_state(res); free(buf); diff --git a/libc/netbsd/net/getnameinfo.c b/libc/netbsd/net/getnameinfo.c index ade5240bd..15d267570 100644 --- a/libc/netbsd/net/getnameinfo.c +++ b/libc/netbsd/net/getnameinfo.c @@ -93,7 +93,7 @@ struct sockinet { }; static int getnameinfo_inet(const struct sockaddr *, socklen_t, char *, - socklen_t, char *, socklen_t, int, const char*); + socklen_t, char *, socklen_t, int, const char*, int); #ifdef INET6 static int ip6_parsenumeric(const struct sockaddr *, const char *, char *, socklen_t, int); @@ -108,16 +108,16 @@ static int getnameinfo_local(const struct sockaddr *, socklen_t, char *, */ int getnameinfo(const struct sockaddr* sa, socklen_t salen, char* host, size_t hostlen, char* serv, size_t servlen, int flags) { - return android_getnameinfoforiface(sa, salen, host, hostlen, serv, servlen, flags, NULL); + return android_getnameinfoforiface(sa, salen, host, hostlen, serv, servlen, flags, NULL, 0); } -int android_getnameinfoforiface(const struct sockaddr* sa, socklen_t salen, char* host, size_t hostlen, char* serv, size_t servlen, int flags, const char* iface) +int android_getnameinfoforiface(const struct sockaddr* sa, socklen_t salen, char* host, size_t hostlen, char* serv, size_t servlen, int flags, const char* iface, int mark) { switch (sa->sa_family) { case AF_INET: case AF_INET6: return getnameinfo_inet(sa, salen, host, hostlen, - serv, servlen, flags, iface); + serv, servlen, flags, iface, mark); case AF_LOCAL: return getnameinfo_local(sa, salen, host, hostlen, serv, servlen, flags); @@ -158,10 +158,10 @@ getnameinfo_local(const struct sockaddr *sa, socklen_t salen, * the address. On failure -1 is returned in which case * normal execution flow shall continue. */ static int -android_gethostbyaddr_proxy(char* nameBuf, size_t nameBufLen, const void *addr, socklen_t addrLen, int addrFamily, const char* iface) +android_gethostbyaddr_proxy(char* nameBuf, size_t nameBufLen, const void *addr, socklen_t addrLen, int addrFamily, const char* iface, int mark) { struct hostent *hostResult = - android_gethostbyaddrforiface_proxy(addr, addrLen, addrFamily, iface); + android_gethostbyaddrforiface_proxy(addr, addrLen, addrFamily, iface, mark); if (hostResult == NULL) return 0; @@ -179,7 +179,7 @@ static int getnameinfo_inet(const struct sockaddr* sa, socklen_t salen, char *host, socklen_t hostlen, char *serv, socklen_t servlen, - int flags, const char* iface) + int flags, const char* iface, int mark) { const struct afd *afd; struct servent *sp; @@ -321,14 +321,15 @@ getnameinfo_inet(const struct sockaddr* sa, socklen_t salen, char android_proxy_buf[MAXDNAME]; int hostnamelen = android_gethostbyaddr_proxy(android_proxy_buf, - MAXDNAME, addr, afd->a_addrlen, afd->a_af, iface); + MAXDNAME, addr, afd->a_addrlen, afd->a_af, iface, mark); if (hostnamelen > 0) { hp = &android_proxy_hostent; hp->h_name = android_proxy_buf; } else if (!hostnamelen) { hp = NULL; } else { - hp = android_gethostbyaddrforiface(addr, afd->a_addrlen, afd->a_af, iface); + hp = android_gethostbyaddrforiface(addr, afd->a_addrlen, afd->a_af, + iface, mark); } if (hp) { diff --git a/libc/netbsd/resolv/res_init.c b/libc/netbsd/resolv/res_init.c index ff6529944..ceb412b85 100644 --- a/libc/netbsd/resolv/res_init.c +++ b/libc/netbsd/resolv/res_init.c @@ -806,4 +806,11 @@ void res_setiface(res_state statp, const char* iface) } } } + +void res_setmark(res_state statp, int mark) +{ + if (statp != NULL) { + statp->_mark = mark; + } +} #endif /* ANDROID_CHANGES */ diff --git a/libc/netbsd/resolv/res_send.c b/libc/netbsd/resolv/res_send.c index 0bb5b6b0c..f65b01584 100644 --- a/libc/netbsd/resolv/res_send.c +++ b/libc/netbsd/resolv/res_send.c @@ -762,10 +762,13 @@ send_vc(res_state statp, if (statp->_vcsock >= 0 && (statp->_flags & RES_F_VC) != 0) { struct sockaddr_storage peer; socklen_t size = sizeof peer; - + int old_mark; + int mark_size = sizeof(old_mark); if (getpeername(statp->_vcsock, (struct sockaddr *)(void *)&peer, &size) < 0 || - !sock_eq((struct sockaddr *)(void *)&peer, nsap)) { + !sock_eq((struct sockaddr *)(void *)&peer, nsap) || + getsockopt(statp->_vcsock, SOL_SOCKET, SO_MARK, &old_mark, &mark_size) < 0 || + old_mark != statp->_mark) { res_nclose(statp); statp->_flags &= ~RES_F_VC; } @@ -795,6 +798,14 @@ send_vc(res_state statp, return (-1); } } + if (statp->_mark != 0) { + if (setsockopt(statp->_vcsock, SOL_SOCKET, + SO_MARK, &statp->_mark, sizeof(statp->_mark)) < 0) { + *terrno = errno; + Perror(statp, stderr, "setsockopt", errno); + return -1; + } + } errno = 0; if (random_bind(statp->_vcsock,nsap->sa_family) < 0) { *terrno = errno; @@ -1070,6 +1081,14 @@ send_dg(res_state statp, return (-1); } } + + if (statp->_mark != 0) { + if (setsockopt(EXT(statp).nssocks[ns], SOL_SOCKET, + SO_MARK, &(statp->_mark), sizeof(statp->_mark)) < 0) { + res_nclose(statp); + return -1; + } + } #ifndef CANNOT_CONNECT_DGRAM /* * On a 4.3BSD+ machine (client and server, @@ -1097,6 +1116,7 @@ send_dg(res_state statp, #endif /* !CANNOT_CONNECT_DGRAM */ Dprint(statp->options & RES_DEBUG, (stdout, ";; new DG socket\n")) + } s = EXT(statp).nssocks[ns]; #ifndef CANNOT_CONNECT_DGRAM diff --git a/libc/private/resolv_private.h b/libc/private/resolv_private.h index 0e799ca2f..61db8c8c0 100644 --- a/libc/private/resolv_private.h +++ b/libc/private/resolv_private.h @@ -175,6 +175,7 @@ struct __res_state { res_send_qhook qhook; /* query hook */ res_send_rhook rhook; /* response hook */ int res_h_errno; /* last one set for this context */ + int _mark; /* If non-0 SET_MARK to _mark on all request sockets */ int _vcsock; /* PRIVATE: for res_send VC i/o */ u_int _flags; /* PRIVATE: see below */ u_int _pad; /* make _u 64 bit aligned */ @@ -496,6 +497,7 @@ int res_getservers(res_state, union res_sockaddr_union *, int); void res_setiface(); +void res_setmark(); u_int res_randomid(void); __END_DECLS From 2fc071797743b88a9a47427d46baed7c7b24f4d2 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Wed, 10 Jul 2013 14:31:03 -0700 Subject: [PATCH 45/77] Add new optimized strlen for arm. This optimized version is primarily targeted at cortex-a15. Tested on all nexus devices using the system/extras/libc_test strlen test. Tested alignments from 1 to 32 that are powers of 2. Tested that strlen does not cross page boundaries at all alignments. Speed improvements listed below: cortex-a15 - Sizes >= 32 bytes, ~75% improvement. - Sizes >= 1024 bytes, ~250% improvement. cortex-a9 - Sizes >= 32 bytes, ~75% improvement. - Sizes >= 1024 bytes, ~85% improvement. krait - Sizes >= 32 bytes, ~95% improvement. - Sizes >= 1024 bytes, ~160% improvement. Change-Id: I361b1a36ed89ab991f2a8f0abbf0d7416d39c8f5 --- libc/arch-arm/arm.mk | 1 - libc/arch-arm/cortex-a15/bionic/strlen.S | 151 ++++++++++++++++++++ libc/arch-arm/cortex-a15/cortex-a15.mk | 1 + libc/arch-arm/cortex-a9/cortex-a9.mk | 2 + libc/arch-arm/{ => generic}/bionic/strlen.c | 0 libc/arch-arm/generic/generic.mk | 1 + libc/arch-arm/krait/krait.mk | 2 + 7 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 libc/arch-arm/cortex-a15/bionic/strlen.S rename libc/arch-arm/{ => generic}/bionic/strlen.c (100%) diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index b308b0520..1a2185f8f 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -15,7 +15,6 @@ _LIBC_ARCH_COMMON_SRC_FILES := \ arch-arm/bionic/setjmp.S \ arch-arm/bionic/sigsetjmp.S \ arch-arm/bionic/strcpy.S \ - arch-arm/bionic/strlen.c.arm \ arch-arm/bionic/syscall.S \ arch-arm/bionic/tgkill.S \ arch-arm/bionic/tkill.S \ diff --git a/libc/arch-arm/cortex-a15/bionic/strlen.S b/libc/arch-arm/cortex-a15/bionic/strlen.S new file mode 100644 index 000000000..d5b8ba422 --- /dev/null +++ b/libc/arch-arm/cortex-a15/bionic/strlen.S @@ -0,0 +1,151 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + + .syntax unified + + .thumb + .thumb_func + +ENTRY(strlen) + pld [r1, #128] + mov r1, r0 + + rsb r3, r0, #0 + ands r3, r3, #7 + beq mainloop + + // Align to a double word (64 bits). + ands ip, r3, #1 + beq align_to_32 + + ldrb r2, [r1], #1 + cmp r2, #0 + beq update_count_and_return + +align_to_32: + ands ip, r3, #2 + beq align_to_64 + + ldrb r2, [r1], #1 + cmp r2, #0 + beq update_count_and_return + ldrb r2, [r1], #1 + cmp r2, #0 + beq update_count_and_return + +align_to_64: + ands ip, r3, #4 + beq mainloop + ldr r3, [r1], #4 + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne zero_in_second_register + +mainloop: + ldrd r2, r3, [r1], #8 + + pld [r1, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne zero_in_second_register + b mainloop + +zero_in_first_register: + sub r1, r1, #4 + +zero_in_second_register: + sub r0, r1, r0 + + // Check for zero in byte 0. + ands r1, ip, #0x80 + beq check_byte1 + + sub r0, r0, #4 + bx lr + +check_byte1: + // Check for zero in byte 1. + ands r1, ip, #0x8000 + beq check_byte2 + + sub r0, r0, #3 + bx lr + +check_byte2: + // Check for zero in byte 2. + ands r1, ip, #0x800000 + beq return + + sub r0, r0, #2 + bx lr + +update_count_and_return: + sub r0, r1, r0 + +return: + sub r0, r0, #1 + bx lr +END(strlen) diff --git a/libc/arch-arm/cortex-a15/cortex-a15.mk b/libc/arch-arm/cortex-a15/cortex-a15.mk index d8193f8c2..0904e6bca 100644 --- a/libc/arch-arm/cortex-a15/cortex-a15.mk +++ b/libc/arch-arm/cortex-a15/cortex-a15.mk @@ -1,5 +1,6 @@ $(call libc-add-cpu-variant-src,MEMCPY,arch-arm/cortex-a15/bionic/memcpy.S) $(call libc-add-cpu-variant-src,MEMSET,arch-arm/cortex-a15/bionic/memset.S) $(call libc-add-cpu-variant-src,STRCMP,arch-arm/cortex-a15/bionic/strcmp.S) +$(call libc-add-cpu-variant-src,STRLEN,arch-arm/cortex-a15/bionic/strlen.S) include bionic/libc/arch-arm/generic/generic.mk diff --git a/libc/arch-arm/cortex-a9/cortex-a9.mk b/libc/arch-arm/cortex-a9/cortex-a9.mk index 48629330a..5c684ed49 100644 --- a/libc/arch-arm/cortex-a9/cortex-a9.mk +++ b/libc/arch-arm/cortex-a9/cortex-a9.mk @@ -1,5 +1,7 @@ $(call libc-add-cpu-variant-src,MEMCPY,arch-arm/cortex-a9/bionic/memcpy.S) $(call libc-add-cpu-variant-src,MEMSET,arch-arm/cortex-a9/bionic/memset.S) $(call libc-add-cpu-variant-src,STRCMP,arch-arm/cortex-a9/bionic/strcmp.S) +# Use cortex-a15 version of strlen. +$(call libc-add-cpu-variant-src,STRLEN,arch-arm/cortex-a15/bionic/strlen.S) include bionic/libc/arch-arm/generic/generic.mk diff --git a/libc/arch-arm/bionic/strlen.c b/libc/arch-arm/generic/bionic/strlen.c similarity index 100% rename from libc/arch-arm/bionic/strlen.c rename to libc/arch-arm/generic/bionic/strlen.c diff --git a/libc/arch-arm/generic/generic.mk b/libc/arch-arm/generic/generic.mk index 358b1e6f6..18cad9da6 100644 --- a/libc/arch-arm/generic/generic.mk +++ b/libc/arch-arm/generic/generic.mk @@ -1,3 +1,4 @@ $(call libc-add-cpu-variant-src,MEMCPY,arch-arm/generic/bionic/memcpy.S) $(call libc-add-cpu-variant-src,MEMSET,arch-arm/generic/bionic/memset.S) $(call libc-add-cpu-variant-src,STRCMP,arch-arm/generic/bionic/strcmp.S) +$(call libc-add-cpu-variant-src,STRLEN,arch-arm/generic/bionic/strlen.c) diff --git a/libc/arch-arm/krait/krait.mk b/libc/arch-arm/krait/krait.mk index 4847f86a8..288afbb61 100644 --- a/libc/arch-arm/krait/krait.mk +++ b/libc/arch-arm/krait/krait.mk @@ -1,5 +1,7 @@ $(call libc-add-cpu-variant-src,MEMCPY,arch-arm/krait/bionic/memcpy.S) $(call libc-add-cpu-variant-src,MEMSET,arch-arm/krait/bionic/memset.S) $(call libc-add-cpu-variant-src,STRCMP,arch-arm/krait/bionic/strcmp.S) +# Use cortex-a15 version of strlen. +$(call libc-add-cpu-variant-src,STRLEN,arch-arm/cortex-a15/bionic/strlen.S) include bionic/libc/arch-arm/generic/generic.mk From 674be7e0fc17cbd813789f81d2de174732632d94 Mon Sep 17 00:00:00 2001 From: Szymon Starzycki Date: Mon, 15 Jul 2013 13:51:56 -0700 Subject: [PATCH 46/77] Kexec header generated from linux header 3.10 Change-Id: Iac26fcc6e0e25905ab52dba91bec16a4fb479d43 --- libc/kernel/common/linux/kexec.h | 24 ------------- libc/kernel/common/uapi/linux/kexec.h | 49 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 24 deletions(-) delete mode 100644 libc/kernel/common/linux/kexec.h create mode 100644 libc/kernel/common/uapi/linux/kexec.h diff --git a/libc/kernel/common/linux/kexec.h b/libc/kernel/common/linux/kexec.h deleted file mode 100644 index 1dfe07ccf..000000000 --- a/libc/kernel/common/linux/kexec.h +++ /dev/null @@ -1,24 +0,0 @@ -/**************************************************************************** - **************************************************************************** - *** - *** This header was automatically generated from a Linux kernel header - *** of the same name, to make information necessary for userspace to - *** call into the kernel available to libc. It contains only constants, - *** structures, and macros generated from the original header, and thus, - *** contains no copyrightable information. - *** - *** To edit the content of this header, modify the corresponding - *** source file (e.g. under external/kernel-headers/original/) then - *** run bionic/libc/kernel/tools/update_all.py - *** - *** Any manual change here will be lost the next time this script will - *** be run. You've been warned! - *** - **************************************************************************** - ****************************************************************************/ -#ifndef LINUX_KEXEC_H -#define LINUX_KEXEC_H -struct pt_regs; -struct task_struct; -/* WARNING: DO NOT EDIT, AUTO-GENERATED CODE - SEE TOP FOR INSTRUCTIONS */ -#endif diff --git a/libc/kernel/common/uapi/linux/kexec.h b/libc/kernel/common/uapi/linux/kexec.h new file mode 100644 index 000000000..977fee6ff --- /dev/null +++ b/libc/kernel/common/uapi/linux/kexec.h @@ -0,0 +1,49 @@ +/**************************************************************************** + **************************************************************************** + *** + *** This header was automatically generated from a Linux kernel header + *** of the same name, to make information necessary for userspace to + *** call into the kernel available to libc. It contains only constants, + *** structures, and macros generated from the original header, and thus, + *** contains no copyrightable information. + *** + *** To edit the content of this header, modify the corresponding + *** source file (e.g. under external/kernel-headers/original/) then + *** run bionic/libc/kernel/tools/update_all.py + *** + *** Any manual change here will be lost the next time this script will + *** be run. You've been warned! + *** + **************************************************************************** + ****************************************************************************/ +#ifndef _UAPILINUX_KEXEC_H +#define _UAPILINUX_KEXEC_H +#include +#define KEXEC_ON_CRASH 0x00000001 +/* WARNING: DO NOT EDIT, AUTO-GENERATED CODE - SEE TOP FOR INSTRUCTIONS */ +#define KEXEC_PRESERVE_CONTEXT 0x00000002 +#define KEXEC_ARCH_MASK 0xffff0000 +#define KEXEC_ARCH_DEFAULT ( 0 << 16) +#define KEXEC_ARCH_386 ( 3 << 16) +/* WARNING: DO NOT EDIT, AUTO-GENERATED CODE - SEE TOP FOR INSTRUCTIONS */ +#define KEXEC_ARCH_X86_64 (62 << 16) +#define KEXEC_ARCH_PPC (20 << 16) +#define KEXEC_ARCH_PPC64 (21 << 16) +#define KEXEC_ARCH_IA_64 (50 << 16) +/* WARNING: DO NOT EDIT, AUTO-GENERATED CODE - SEE TOP FOR INSTRUCTIONS */ +#define KEXEC_ARCH_ARM (40 << 16) +#define KEXEC_ARCH_S390 (22 << 16) +#define KEXEC_ARCH_SH (42 << 16) +#define KEXEC_ARCH_MIPS_LE (10 << 16) +/* WARNING: DO NOT EDIT, AUTO-GENERATED CODE - SEE TOP FOR INSTRUCTIONS */ +#define KEXEC_ARCH_MIPS ( 8 << 16) +#define KEXEC_SEGMENT_MAX 16 +struct kexec_segment { + const void *buf; +/* WARNING: DO NOT EDIT, AUTO-GENERATED CODE - SEE TOP FOR INSTRUCTIONS */ + size_t bufsz; + const void *mem; + size_t memsz; +}; +/* WARNING: DO NOT EDIT, AUTO-GENERATED CODE - SEE TOP FOR INSTRUCTIONS */ +#endif From d8d10a8994472e40d19301b7087806630877b4d5 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Mon, 15 Jul 2013 13:55:45 -0700 Subject: [PATCH 47/77] Fix assembler errors in generic arm strlen.c. Tested using a static version of the strlen libc_test program on a nexus7 that uses the generic code. Change-Id: If04d15dcb6c0b18f27f2fefadca5510ed49016c5 --- libc/arch-arm/generic/bionic/strlen.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/libc/arch-arm/generic/bionic/strlen.c b/libc/arch-arm/generic/bionic/strlen.c index ca0669de3..824cf78d0 100644 --- a/libc/arch-arm/generic/bionic/strlen.c +++ b/libc/arch-arm/generic/bionic/strlen.c @@ -33,16 +33,16 @@ size_t strlen(const char *s) { __builtin_prefetch(s); __builtin_prefetch(s+32); - + union { const char *b; const uint32_t *w; uintptr_t i; } u; - + // these are some scratch variables for the asm code below uint32_t v, t; - + // initialize the string length to zero size_t l = 0; @@ -69,42 +69,50 @@ size_t strlen(const char *s) "sub %[t], %[v], %[mask], lsr #7\n" "and %[t], %[t], %[mask] \n" "bics %[t], %[t], %[v] \n" + "it eq \n" "ldreq %[v], [%[s]], #4 \n" #if !defined(__OPTIMIZE_SIZE__) "bne 1f \n" "sub %[t], %[v], %[mask], lsr #7\n" "and %[t], %[t], %[mask] \n" "bics %[t], %[t], %[v] \n" + "it eq \n" "ldreq %[v], [%[s]], #4 \n" "bne 1f \n" "sub %[t], %[v], %[mask], lsr #7\n" "and %[t], %[t], %[mask] \n" "bics %[t], %[t], %[v] \n" + "it eq \n" "ldreq %[v], [%[s]], #4 \n" "bne 1f \n" "sub %[t], %[v], %[mask], lsr #7\n" "and %[t], %[t], %[mask] \n" "bics %[t], %[t], %[v] \n" + "it eq \n" "ldreq %[v], [%[s]], #4 \n" "bne 1f \n" "sub %[t], %[v], %[mask], lsr #7\n" "and %[t], %[t], %[mask] \n" "bics %[t], %[t], %[v] \n" + "it eq \n" "ldreq %[v], [%[s]], #4 \n" "bne 1f \n" "sub %[t], %[v], %[mask], lsr #7\n" "and %[t], %[t], %[mask] \n" "bics %[t], %[t], %[v] \n" + "it eq \n" "ldreq %[v], [%[s]], #4 \n" "bne 1f \n" "sub %[t], %[v], %[mask], lsr #7\n" "and %[t], %[t], %[mask] \n" "bics %[t], %[t], %[v] \n" + "it eq \n" "ldreq %[v], [%[s]], #4 \n" "bne 1f \n" "sub %[t], %[v], %[mask], lsr #7\n" "and %[t], %[t], %[mask] \n" "bics %[t], %[t], %[v] \n" + "it eq \n" "ldreq %[v], [%[s]], #4 \n" #endif "beq 0b \n" @@ -117,13 +125,14 @@ size_t strlen(const char *s) "beq 2f \n" "add %[l], %[l], #1 \n" "tst %[v], #0xFF0000 \n" + "it ne \n" "addne %[l], %[l], #1 \n" "2: \n" : [l]"=&r"(l), [v]"=&r"(v), [t]"=&r"(t), [s]"=&r"(u.b) : "%[l]"(l), "%[s]"(u.b), [mask]"r"(0x80808080UL) : "cc" ); - + done: return l; } From d119b7b6f48fe507088cfb98bcafa99b320fd884 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Mon, 15 Jul 2013 12:49:26 -0700 Subject: [PATCH 48/77] Optimize strcat/strcpy, small tweaks to strlen. Create one version of strcat/strcpy/strlen for cortex-a15/krait and another version for cortex-a9. Tested with the libc_test strcat/strcpy/strlen tests. Including new tests that verify that the src for strcat/strcpy do not overread across page boundaries. NOTE: The handling of unaligned strcpy (same code in strcat) could probably be optimized further such that the src is read 64 bits at a time instead of the partial reads occurring now. strlen improves slightly since it was recently optimized. Performance improvements for strcpy and strcat (using an empty dest string): cortex-a9 - Small copies vary from about 5% to 20% as the size gets above 10 bytes. - Copies >= 1024, about a 60% improvement. - Unaligned copies, from about 40% improvement. cortex-a15 - Most small copies exhibit a 100% improvement, a few copies only improve by 20%. - Copies >= 1024, about 150% improvement. - Unaligned copies, about 100% improvement. krait - Most small copies vary widely, but on average 20% improvement, then the performance gets better, hitting about a 100% improvement when copies 64 bytes of data. - Copies >= 1024, about 100% improvement. - When coping MBs of data, about 50% improvement. - Unaligned copies, about 90% improvement. As strcat destination strings get larger in size: cortex-a9 - about 40% improvement for small dst strings (>= 32). - about 250% improvement for dst strings >= 1024. cortex-a15 - about 200% improvement for small dst strings (>=32). - about 250% improvement for dst strings >= 1024. krait - about 25% improvement for small dst strings (>=32). - about 100% improvement for dst strings >=1024. Change-Id: Ifd091ebdbce70fe35a7c5d8f71d5914255f3af35 --- libc/Android.mk | 1 - libc/arch-arm/arm.mk | 1 - libc/arch-arm/cortex-a15/bionic/strcat.S | 568 ++++++++++++++++++++ libc/arch-arm/cortex-a15/bionic/strcpy.S | 451 ++++++++++++++++ libc/arch-arm/cortex-a15/bionic/strlen.S | 80 +-- libc/arch-arm/cortex-a15/cortex-a15.mk | 2 + libc/arch-arm/cortex-a9/bionic/strcat.S | 548 +++++++++++++++++++ libc/arch-arm/cortex-a9/bionic/strcpy.S | 456 ++++++++++++++++ libc/arch-arm/cortex-a9/bionic/strlen.S | 167 ++++++ libc/arch-arm/cortex-a9/cortex-a9.mk | 5 +- libc/arch-arm/{ => generic}/bionic/strcpy.S | 0 libc/arch-arm/generic/generic.mk | 2 + libc/arch-arm/krait/krait.mk | 4 +- 13 files changed, 2247 insertions(+), 38 deletions(-) create mode 100644 libc/arch-arm/cortex-a15/bionic/strcat.S create mode 100644 libc/arch-arm/cortex-a15/bionic/strcpy.S create mode 100644 libc/arch-arm/cortex-a9/bionic/strcat.S create mode 100644 libc/arch-arm/cortex-a9/bionic/strcpy.S create mode 100644 libc/arch-arm/cortex-a9/bionic/strlen.S rename libc/arch-arm/{ => generic}/bionic/strcpy.S (100%) diff --git a/libc/Android.mk b/libc/Android.mk index db668d379..f353c416f 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -357,7 +357,6 @@ libc_common_src_files += \ bionic/memmove.c.arm \ string/bcopy.c \ string/strncmp.c \ - string/strcat.c \ string/strncat.c \ string/strncpy.c \ bionic/strchr.cpp \ diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index 1a2185f8f..7fb15a1c7 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -14,7 +14,6 @@ _LIBC_ARCH_COMMON_SRC_FILES := \ arch-arm/bionic/_setjmp.S \ arch-arm/bionic/setjmp.S \ arch-arm/bionic/sigsetjmp.S \ - arch-arm/bionic/strcpy.S \ arch-arm/bionic/syscall.S \ arch-arm/bionic/tgkill.S \ arch-arm/bionic/tkill.S \ diff --git a/libc/arch-arm/cortex-a15/bionic/strcat.S b/libc/arch-arm/cortex-a15/bionic/strcat.S new file mode 100644 index 000000000..72d4e9eb0 --- /dev/null +++ b/libc/arch-arm/cortex-a15/bionic/strcat.S @@ -0,0 +1,568 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + + .syntax unified + + .thumb + .thumb_func + + .macro m_push + push {r0, r4, r5, lr} + .endm // m_push + + .macro m_pop + pop {r0, r4, r5, pc} + .endm // m_pop + + .macro m_scan_byte + ldrb r3, [r0] + cbz r3, strcat_r0_scan_done + add r0, #1 + .endm // m_scan_byte + + .macro m_copy_byte reg, cmd, label + ldrb \reg, [r1], #1 + strb \reg, [r0], #1 + \cmd \reg, \label + .endm // m_copy_byte + +ENTRY(strcat) + // Quick check to see if src is empty. + ldrb r2, [r1] + pld [r1, #0] + cbnz r2, strcat_continue + bx lr + +strcat_continue: + // To speed up really small dst strings, unroll checking the first 4 bytes. + m_push + m_scan_byte + m_scan_byte + m_scan_byte + m_scan_byte + + ands r3, r0, #7 + beq strcat_mainloop + + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq strcat_align_to_32 + + ldrb r5, [r0] + cbz r5, strcat_r0_scan_done + add r0, r0, #1 + +strcat_align_to_32: + bcc strcat_align_to_64 + + ldrb r2, [r0] + cbz r2, strcat_r0_scan_done + add r0, r0, #1 + ldrb r4, [r0] + cbz r4, strcat_r0_scan_done + add r0, r0, #1 + +strcat_align_to_64: + tst r3, #4 + beq strcat_mainloop + ldr r3, [r0], #4 + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcat_zero_in_second_register + b strcat_mainloop + +strcat_r0_scan_done: + // For short copies, hard-code checking the first 8 bytes since this + // new code doesn't win until after about 8 bytes. + m_copy_byte reg=r2, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r3, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r4, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r5, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r2, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r3, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r4, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r5, cmd=cbnz, label=strcpy_continue + +strcpy_finish: + m_pop + +strcpy_continue: + ands r3, r0, #7 + beq strcpy_check_src_align + + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq strcpy_align_to_32 + + ldrb r2, [r1], #1 + strb r2, [r0], #1 + cbz r2, strcpy_complete + +strcpy_align_to_32: + bcc strcpy_align_to_64 + + ldrb r2, [r1], #1 + strb r2, [r0], #1 + cbz r2, strcpy_complete + ldrb r2, [r1], #1 + strb r2, [r0], #1 + cbz r2, strcpy_complete + +strcpy_align_to_64: + tst r3, #4 + beq strcpy_check_src_align + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + str r2, [r0], #4 + +strcpy_check_src_align: + // At this point dst is aligned to a double word, check if src + // is also aligned to a double word. + ands r3, r1, #7 + bne strcpy_unaligned_copy + + .p2align 2 +strcpy_mainloop: + ldrd r2, r3, [r1], #8 + + pld [r1, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + strd r2, r3, [r0], #8 + b strcpy_mainloop + +strcpy_complete: + m_pop + +strcpy_zero_in_first_register: + lsls lr, ip, #17 + bne strcpy_copy1byte + bcs strcpy_copy2bytes + lsls ip, ip, #1 + bne strcpy_copy3bytes + +strcpy_copy4bytes: + // Copy 4 bytes to the destiniation. + str r2, [r0] + m_pop + +strcpy_copy1byte: + strb r2, [r0] + m_pop + +strcpy_copy2bytes: + strh r2, [r0] + m_pop + +strcpy_copy3bytes: + strh r2, [r0], #2 + lsr r2, #16 + strb r2, [r0] + m_pop + +strcpy_zero_in_second_register: + lsls lr, ip, #17 + bne strcpy_copy5bytes + bcs strcpy_copy6bytes + lsls ip, ip, #1 + bne strcpy_copy7bytes + + // Copy 8 bytes to the destination. + strd r2, r3, [r0] + m_pop + +strcpy_copy5bytes: + str r2, [r0], #4 + strb r3, [r0] + m_pop + +strcpy_copy6bytes: + str r2, [r0], #4 + strh r3, [r0] + m_pop + +strcpy_copy7bytes: + str r2, [r0], #4 + strh r3, [r0], #2 + lsr r3, #16 + strb r3, [r0] + m_pop + +strcpy_unaligned_copy: + // Dst is aligned to a double word, while src is at an unknown alignment. + // There are 7 different versions of the unaligned copy code + // to prevent overreading the src. The mainloop of every single version + // will store 64 bits per loop. The difference is how much of src can + // be read without potentially crossing a page boundary. + tbb [pc, r3] +strcpy_unaligned_branchtable: + .byte 0 + .byte ((strcpy_unalign7 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign6 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign5 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign4 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign3 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign2 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign1 - strcpy_unaligned_branchtable)/2) + + .p2align 2 + // Can read 7 bytes before possibly crossing a page. +strcpy_unalign7: + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + ldrb r3, [r1] + cbz r3, strcpy_unalign7_copy5bytes + ldrb r4, [r1, #1] + cbz r4, strcpy_unalign7_copy6bytes + ldrb r5, [r1, #2] + cbz r5, strcpy_unalign7_copy7bytes + + ldr r3, [r1], #4 + pld [r1, #64] + + lsrs ip, r3, #24 + strd r2, r3, [r0], #8 + beq strcpy_unalign_return + b strcpy_unalign7 + +strcpy_unalign7_copy5bytes: + str r2, [r0], #4 + strb r3, [r0] +strcpy_unalign_return: + m_pop + +strcpy_unalign7_copy6bytes: + str r2, [r0], #4 + strb r3, [r0], #1 + strb r4, [r0], #1 + m_pop + +strcpy_unalign7_copy7bytes: + str r2, [r0], #4 + strb r3, [r0], #1 + strb r4, [r0], #1 + strb r5, [r0], #1 + m_pop + + .p2align 2 + // Can read 6 bytes before possibly crossing a page. +strcpy_unalign6: + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + ldrb r4, [r1] + cbz r4, strcpy_unalign_copy5bytes + ldrb r5, [r1, #1] + cbz r5, strcpy_unalign_copy6bytes + + ldr r3, [r1], #4 + pld [r1, #64] + + tst r3, #0xff0000 + beq strcpy_copy7bytes + lsrs ip, r3, #24 + strd r2, r3, [r0], #8 + beq strcpy_unalign_return + b strcpy_unalign6 + + .p2align 2 + // Can read 5 bytes before possibly crossing a page. +strcpy_unalign5: + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + ldrb r4, [r1] + cbz r4, strcpy_unalign_copy5bytes + + ldr r3, [r1], #4 + + pld [r1, #64] + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + strd r2, r3, [r0], #8 + b strcpy_unalign5 + +strcpy_unalign_copy5bytes: + str r2, [r0], #4 + strb r4, [r0] + m_pop + +strcpy_unalign_copy6bytes: + str r2, [r0], #4 + strb r4, [r0], #1 + strb r5, [r0] + m_pop + + .p2align 2 + // Can read 4 bytes before possibly crossing a page. +strcpy_unalign4: + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + ldr r3, [r1], #4 + pld [r1, #64] + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + strd r2, r3, [r0], #8 + b strcpy_unalign4 + + .p2align 2 + // Can read 3 bytes before possibly crossing a page. +strcpy_unalign3: + ldrb r2, [r1] + cbz r2, strcpy_unalign3_copy1byte + ldrb r3, [r1, #1] + cbz r3, strcpy_unalign3_copy2bytes + ldrb r4, [r1, #2] + cbz r4, strcpy_unalign3_copy3bytes + + ldr r2, [r1], #4 + ldr r3, [r1], #4 + + pld [r1, #64] + + lsrs lr, r2, #24 + beq strcpy_copy4bytes + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + strd r2, r3, [r0], #8 + b strcpy_unalign3 + +strcpy_unalign3_copy1byte: + strb r2, [r0] + m_pop + +strcpy_unalign3_copy2bytes: + strb r2, [r0], #1 + strb r3, [r0] + m_pop + +strcpy_unalign3_copy3bytes: + strb r2, [r0], #1 + strb r3, [r0], #1 + strb r4, [r0] + m_pop + + .p2align 2 + // Can read 2 bytes before possibly crossing a page. +strcpy_unalign2: + ldrb r2, [r1] + cbz r2, strcpy_unalign_copy1byte + ldrb r4, [r1, #1] + cbz r4, strcpy_unalign_copy2bytes + + ldr r2, [r1], #4 + ldr r3, [r1], #4 + pld [r1, #64] + + tst r2, #0xff0000 + beq strcpy_copy3bytes + lsrs ip, r2, #24 + beq strcpy_copy4bytes + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + strd r2, r3, [r0], #8 + b strcpy_unalign2 + + .p2align 2 + // Can read 1 byte before possibly crossing a page. +strcpy_unalign1: + ldrb r2, [r1] + cbz r2, strcpy_unalign_copy1byte + + ldr r2, [r1], #4 + ldr r3, [r1], #4 + + pld [r1, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + strd r2, r3, [r0], #8 + b strcpy_unalign1 + +strcpy_unalign_copy1byte: + strb r2, [r0] + m_pop + +strcpy_unalign_copy2bytes: + strb r2, [r0], #1 + strb r4, [r0] + m_pop + + .p2align 2 +strcat_mainloop: + ldrd r2, r3, [r0], #8 + + pld [r0, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcat_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcat_zero_in_second_register + b strcat_mainloop + +strcat_zero_in_first_register: + // Prefetch the src now, it's going to be used soon. + pld [r1, #0] + lsls lr, ip, #17 + bne strcat_sub8 + bcs strcat_sub7 + lsls ip, ip, #1 + bne strcat_sub6 + + sub r0, r0, #5 + b strcat_r0_scan_done + +strcat_sub8: + sub r0, r0, #8 + b strcat_r0_scan_done + +strcat_sub7: + sub r0, r0, #7 + b strcat_r0_scan_done + +strcat_sub6: + sub r0, r0, #6 + b strcat_r0_scan_done + +strcat_zero_in_second_register: + // Prefetch the src now, it's going to be used soon. + pld [r1, #0] + lsls lr, ip, #17 + bne strcat_sub4 + bcs strcat_sub3 + lsls ip, ip, #1 + bne strcat_sub2 + + sub r0, r0, #1 + b strcat_r0_scan_done + +strcat_sub4: + sub r0, r0, #4 + b strcat_r0_scan_done + +strcat_sub3: + sub r0, r0, #3 + b strcat_r0_scan_done + +strcat_sub2: + sub r0, r0, #2 + b strcat_r0_scan_done +END(strcat) diff --git a/libc/arch-arm/cortex-a15/bionic/strcpy.S b/libc/arch-arm/cortex-a15/bionic/strcpy.S new file mode 100644 index 000000000..577354034 --- /dev/null +++ b/libc/arch-arm/cortex-a15/bionic/strcpy.S @@ -0,0 +1,451 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + + .syntax unified + + .thumb + .thumb_func + + .macro m_push + push {r0, r4, r5, lr} + .endm // m_push + + .macro m_pop + pop {r0, r4, r5, pc} + .endm // m_pop + + .macro m_copy_byte reg, cmd, label + ldrb \reg, [r1], #1 + strb \reg, [r0], #1 + \cmd \reg, \label + .endm // m_copy_byte + +ENTRY(strcpy) + // For short copies, hard-code checking the first 8 bytes since this + // new code doesn't win until after about 8 bytes. + m_push + m_copy_byte reg=r2, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r3, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r4, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r5, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r2, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r3, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r4, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r5, cmd=cbnz, label=strcpy_continue + +strcpy_finish: + m_pop + +strcpy_continue: + pld [r1, #0] + ands r3, r0, #7 + beq strcpy_check_src_align + + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq strcpy_align_to_32 + + ldrb r2, [r1], #1 + strb r2, [r0], #1 + cbz r2, strcpy_complete + +strcpy_align_to_32: + bcc strcpy_align_to_64 + + ldrb r2, [r1], #1 + strb r2, [r0], #1 + cbz r2, strcpy_complete + ldrb r2, [r1], #1 + strb r2, [r0], #1 + cbz r2, strcpy_complete + +strcpy_align_to_64: + tst r3, #4 + beq strcpy_check_src_align + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + str r2, [r0], #4 + +strcpy_check_src_align: + // At this point dst is aligned to a double word, check if src + // is also aligned to a double word. + ands r3, r1, #7 + bne strcpy_unaligned_copy + + .p2align 2 +strcpy_mainloop: + ldrd r2, r3, [r1], #8 + + pld [r1, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + strd r2, r3, [r0], #8 + b strcpy_mainloop + +strcpy_complete: + m_pop + +strcpy_zero_in_first_register: + lsls lr, ip, #17 + bne strcpy_copy1byte + bcs strcpy_copy2bytes + lsls ip, ip, #1 + bne strcpy_copy3bytes + +strcpy_copy4bytes: + // Copy 4 bytes to the destiniation. + str r2, [r0] + m_pop + +strcpy_copy1byte: + strb r2, [r0] + m_pop + +strcpy_copy2bytes: + strh r2, [r0] + m_pop + +strcpy_copy3bytes: + strh r2, [r0], #2 + lsr r2, #16 + strb r2, [r0] + m_pop + +strcpy_zero_in_second_register: + lsls lr, ip, #17 + bne strcpy_copy5bytes + bcs strcpy_copy6bytes + lsls ip, ip, #1 + bne strcpy_copy7bytes + + // Copy 8 bytes to the destination. + strd r2, r3, [r0] + m_pop + +strcpy_copy5bytes: + str r2, [r0], #4 + strb r3, [r0] + m_pop + +strcpy_copy6bytes: + str r2, [r0], #4 + strh r3, [r0] + m_pop + +strcpy_copy7bytes: + str r2, [r0], #4 + strh r3, [r0], #2 + lsr r3, #16 + strb r3, [r0] + m_pop + +strcpy_unaligned_copy: + // Dst is aligned to a double word, while src is at an unknown alignment. + // There are 7 different versions of the unaligned copy code + // to prevent overreading the src. The mainloop of every single version + // will store 64 bits per loop. The difference is how much of src can + // be read without potentially crossing a page boundary. + tbb [pc, r3] +strcpy_unaligned_branchtable: + .byte 0 + .byte ((strcpy_unalign7 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign6 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign5 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign4 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign3 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign2 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign1 - strcpy_unaligned_branchtable)/2) + + .p2align 2 + // Can read 7 bytes before possibly crossing a page. +strcpy_unalign7: + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + ldrb r3, [r1] + cbz r3, strcpy_unalign7_copy5bytes + ldrb r4, [r1, #1] + cbz r4, strcpy_unalign7_copy6bytes + ldrb r5, [r1, #2] + cbz r5, strcpy_unalign7_copy7bytes + + ldr r3, [r1], #4 + pld [r1, #64] + + lsrs ip, r3, #24 + strd r2, r3, [r0], #8 + beq strcpy_unalign_return + b strcpy_unalign7 + +strcpy_unalign7_copy5bytes: + str r2, [r0], #4 + strb r3, [r0] +strcpy_unalign_return: + m_pop + +strcpy_unalign7_copy6bytes: + str r2, [r0], #4 + strb r3, [r0], #1 + strb r4, [r0], #1 + m_pop + +strcpy_unalign7_copy7bytes: + str r2, [r0], #4 + strb r3, [r0], #1 + strb r4, [r0], #1 + strb r5, [r0], #1 + m_pop + + .p2align 2 + // Can read 6 bytes before possibly crossing a page. +strcpy_unalign6: + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + ldrb r4, [r1] + cbz r4, strcpy_unalign_copy5bytes + ldrb r5, [r1, #1] + cbz r5, strcpy_unalign_copy6bytes + + ldr r3, [r1], #4 + pld [r1, #64] + + tst r3, #0xff0000 + beq strcpy_copy7bytes + lsrs ip, r3, #24 + strd r2, r3, [r0], #8 + beq strcpy_unalign_return + b strcpy_unalign6 + + .p2align 2 + // Can read 5 bytes before possibly crossing a page. +strcpy_unalign5: + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + ldrb r4, [r1] + cbz r4, strcpy_unalign_copy5bytes + + ldr r3, [r1], #4 + + pld [r1, #64] + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + strd r2, r3, [r0], #8 + b strcpy_unalign5 + +strcpy_unalign_copy5bytes: + str r2, [r0], #4 + strb r4, [r0] + m_pop + +strcpy_unalign_copy6bytes: + str r2, [r0], #4 + strb r4, [r0], #1 + strb r5, [r0] + m_pop + + .p2align 2 + // Can read 4 bytes before possibly crossing a page. +strcpy_unalign4: + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + ldr r3, [r1], #4 + pld [r1, #64] + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + strd r2, r3, [r0], #8 + b strcpy_unalign4 + + .p2align 2 + // Can read 3 bytes before possibly crossing a page. +strcpy_unalign3: + ldrb r2, [r1] + cbz r2, strcpy_unalign3_copy1byte + ldrb r3, [r1, #1] + cbz r3, strcpy_unalign3_copy2bytes + ldrb r4, [r1, #2] + cbz r4, strcpy_unalign3_copy3bytes + + ldr r2, [r1], #4 + ldr r3, [r1], #4 + + pld [r1, #64] + + lsrs lr, r2, #24 + beq strcpy_copy4bytes + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + strd r2, r3, [r0], #8 + b strcpy_unalign3 + +strcpy_unalign3_copy1byte: + strb r2, [r0] + m_pop + +strcpy_unalign3_copy2bytes: + strb r2, [r0], #1 + strb r3, [r0] + m_pop + +strcpy_unalign3_copy3bytes: + strb r2, [r0], #1 + strb r3, [r0], #1 + strb r4, [r0] + m_pop + + .p2align 2 + // Can read 2 bytes before possibly crossing a page. +strcpy_unalign2: + ldrb r2, [r1] + cbz r2, strcpy_unalign_copy1byte + ldrb r4, [r1, #1] + cbz r4, strcpy_unalign_copy2bytes + + ldr r2, [r1], #4 + ldr r3, [r1], #4 + pld [r1, #64] + + tst r2, #0xff0000 + beq strcpy_copy3bytes + lsrs ip, r2, #24 + beq strcpy_copy4bytes + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + strd r2, r3, [r0], #8 + b strcpy_unalign2 + + .p2align 2 + // Can read 1 byte before possibly crossing a page. +strcpy_unalign1: + ldrb r2, [r1] + cbz r2, strcpy_unalign_copy1byte + + ldr r2, [r1], #4 + ldr r3, [r1], #4 + + pld [r1, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + strd r2, r3, [r0], #8 + b strcpy_unalign1 + +strcpy_unalign_copy1byte: + strb r2, [r0] + m_pop + +strcpy_unalign_copy2bytes: + strb r2, [r0], #1 + strb r4, [r0] + m_pop +END(strcpy) diff --git a/libc/arch-arm/cortex-a15/bionic/strlen.S b/libc/arch-arm/cortex-a15/bionic/strlen.S index d5b8ba422..08f6d193b 100644 --- a/libc/arch-arm/cortex-a15/bionic/strlen.S +++ b/libc/arch-arm/cortex-a15/bionic/strlen.S @@ -61,34 +61,32 @@ .thumb_func ENTRY(strlen) - pld [r1, #128] - mov r1, r0 + pld [r0, #0] + mov r1, r0 - rsb r3, r0, #0 - ands r3, r3, #7 + ands r3, r0, #7 beq mainloop // Align to a double word (64 bits). - ands ip, r3, #1 + rsb r3, r3, #8 + lsls ip, r3, #31 beq align_to_32 ldrb r2, [r1], #1 - cmp r2, #0 - beq update_count_and_return + cbz r2, update_count_and_return align_to_32: + bcc align_to_64 ands ip, r3, #2 beq align_to_64 ldrb r2, [r1], #1 - cmp r2, #0 - beq update_count_and_return + cbz r2, update_count_and_return ldrb r2, [r1], #1 - cmp r2, #0 - beq update_count_and_return + cbz r2, update_count_and_return align_to_64: - ands ip, r3, #4 + tst r3, #4 beq mainloop ldr r3, [r1], #4 @@ -97,6 +95,7 @@ align_to_64: ands ip, ip, #0x80808080 bne zero_in_second_register + .p2align 2 mainloop: ldrd r2, r3, [r1], #8 @@ -113,39 +112,54 @@ mainloop: bne zero_in_second_register b mainloop +update_count_and_return: + sub r0, r1, r0 + sub r0, r0, #1 + bx lr + zero_in_first_register: - sub r1, r1, #4 + sub r0, r1, r0 + lsls r3, ip, #17 + bne sub8_and_return + bcs sub7_and_return + lsls ip, ip, #1 + bne sub6_and_return + + sub r0, r0, #5 + bx lr + +sub8_and_return: + sub r0, r0, #8 + bx lr + +sub7_and_return: + sub r0, r0, #7 + bx lr + +sub6_and_return: + sub r0, r0, #6 + bx lr zero_in_second_register: sub r0, r1, r0 + lsls r3, ip, #17 + bne sub4_and_return + bcs sub3_and_return + lsls ip, ip, #1 + bne sub2_and_return - // Check for zero in byte 0. - ands r1, ip, #0x80 - beq check_byte1 + sub r0, r0, #1 + bx lr +sub4_and_return: sub r0, r0, #4 bx lr -check_byte1: - // Check for zero in byte 1. - ands r1, ip, #0x8000 - beq check_byte2 - +sub3_and_return: sub r0, r0, #3 bx lr -check_byte2: - // Check for zero in byte 2. - ands r1, ip, #0x800000 - beq return - +sub2_and_return: sub r0, r0, #2 bx lr - -update_count_and_return: - sub r0, r1, r0 - -return: - sub r0, r0, #1 - bx lr END(strlen) diff --git a/libc/arch-arm/cortex-a15/cortex-a15.mk b/libc/arch-arm/cortex-a15/cortex-a15.mk index 0904e6bca..281e424ba 100644 --- a/libc/arch-arm/cortex-a15/cortex-a15.mk +++ b/libc/arch-arm/cortex-a15/cortex-a15.mk @@ -1,6 +1,8 @@ $(call libc-add-cpu-variant-src,MEMCPY,arch-arm/cortex-a15/bionic/memcpy.S) $(call libc-add-cpu-variant-src,MEMSET,arch-arm/cortex-a15/bionic/memset.S) +$(call libc-add-cpu-variant-src,STRCAT,arch-arm/cortex-a15/bionic/strcat.S) $(call libc-add-cpu-variant-src,STRCMP,arch-arm/cortex-a15/bionic/strcmp.S) +$(call libc-add-cpu-variant-src,STRCPY,arch-arm/cortex-a15/bionic/strcpy.S) $(call libc-add-cpu-variant-src,STRLEN,arch-arm/cortex-a15/bionic/strlen.S) include bionic/libc/arch-arm/generic/generic.mk diff --git a/libc/arch-arm/cortex-a9/bionic/strcat.S b/libc/arch-arm/cortex-a9/bionic/strcat.S new file mode 100644 index 000000000..0f5baef4c --- /dev/null +++ b/libc/arch-arm/cortex-a9/bionic/strcat.S @@ -0,0 +1,548 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + + .syntax unified + + .thumb + .thumb_func + + .macro m_push + push {r0, r4, r5, lr} + .endm // m_push + + .macro m_ret inst + \inst {r0, r4, r5, pc} + .endm // m_ret + + .macro m_scan_byte + ldrb r3, [r0] + cbz r3, strcat_r0_scan_done + add r0, #1 + .endm // m_scan_byte + + .macro m_copy_byte reg, cmd, label + ldrb \reg, [r1], #1 + strb \reg, [r0], #1 + \cmd \reg, \label + .endm // m_copy_byte + +ENTRY(strcat) + // Quick check to see if src is empty. + ldrb r2, [r1] + pld [r1, #0] + cbnz r2, strcat_continue + bx lr + +strcat_continue: + // To speed up really small dst strings, unroll checking the first 4 bytes. + m_push + m_scan_byte + m_scan_byte + m_scan_byte + m_scan_byte + + ands r3, r0, #7 + bne strcat_align_src + + .p2align 2 +strcat_mainloop: + ldmia r0!, {r2, r3} + + pld [r0, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcat_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcat_zero_in_second_register + b strcat_mainloop + +strcat_zero_in_first_register: + sub r0, r0, #4 + +strcat_zero_in_second_register: + // Check for zero in byte 0. + tst ip, #0x80 + it ne + subne r0, r0, #4 + bne strcat_r0_scan_done + // Check for zero in byte 1. + tst ip, #0x8000 + it ne + subne r0, r0, #3 + bne strcat_r0_scan_done + // Check for zero in byte 2. + tst ip, #0x800000 + it ne + subne r0, r0, #2 + it eq + // Zero is in byte 3. + subeq r0, r0, #1 + +strcat_r0_scan_done: + // Unroll the first 8 bytes that will be copied. + m_copy_byte reg=r2, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r3, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r4, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r5, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r2, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r3, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r4, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r5, cmd=cbnz, label=strcpy_continue + +strcpy_finish: + m_ret inst=pop + +strcpy_continue: + pld [r1, #0] + ands r3, r0, #7 + bne strcpy_align_dst + +strcpy_check_src_align: + // At this point dst is aligned to a double word, check if src + // is also aligned to a double word. + ands r3, r1, #7 + bne strcpy_unaligned_copy + + .p2align 2 +strcpy_mainloop: + ldmia r1!, {r2, r3} + + pld [r1, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + stmia r0!, {r2, r3} + b strcpy_mainloop + +strcpy_zero_in_first_register: + lsls lr, ip, #17 + itt ne + strbne r2, [r0] + m_ret inst=popne + itt cs + strhcs r2, [r0] + m_ret inst=popcs + lsls ip, ip, #1 + itt eq + streq r2, [r0] + m_ret inst=popeq + strh r2, [r0], #2 + lsr r3, r2, #16 + strb r3, [r0] + m_ret inst=pop + +strcpy_zero_in_second_register: + lsls lr, ip, #17 + ittt ne + stmiane r0!, {r2} + strbne r3, [r0] + m_ret inst=popne + ittt cs + strcs r2, [r0], #4 + strhcs r3, [r0] + m_ret inst=popcs + lsls ip, ip, #1 + itt eq + stmiaeq r0, {r2, r3} + m_ret inst=popeq + stmia r0!, {r2} + strh r3, [r0], #2 + lsr r4, r3, #16 + strb r4, [r0] + m_ret inst=pop + +strcpy_align_dst: + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq strcpy_align_to_32 + + ldrb r2, [r1], #1 + strb r2, [r0], #1 + cbz r2, strcpy_complete + +strcpy_align_to_32: + bcc strcpy_align_to_64 + + ldrb r4, [r1], #1 + strb r4, [r0], #1 + cmp r4, #0 + it eq + m_ret inst=popeq + ldrb r5, [r1], #1 + strb r5, [r0], #1 + cmp r5, #0 + it eq + m_ret inst=popeq + +strcpy_align_to_64: + tst r3, #4 + beq strcpy_check_src_align + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + stmia r0!, {r2} + b strcpy_check_src_align + +strcpy_complete: + m_ret inst=pop + +strcpy_unaligned_copy: + // Dst is aligned to a double word, while src is at an unknown alignment. + // There are 7 different versions of the unaligned copy code + // to prevent overreading the src. The mainloop of every single version + // will store 64 bits per loop. The difference is how much of src can + // be read without potentially crossing a page boundary. + tbb [pc, r3] +strcpy_unaligned_branchtable: + .byte 0 + .byte ((strcpy_unalign7 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign6 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign5 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign4 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign3 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign2 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign1 - strcpy_unaligned_branchtable)/2) + + .p2align 2 + // Can read 7 bytes before possibly crossing a page. +strcpy_unalign7: + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + ldrb r3, [r1] + cbz r3, strcpy_unalign7_copy5bytes + ldrb r4, [r1, #1] + cbz r4, strcpy_unalign7_copy6bytes + ldrb r5, [r1, #2] + cbz r5, strcpy_unalign7_copy7bytes + + ldr r3, [r1], #4 + pld [r1, #64] + + lsrs ip, r3, #24 + stmia r0!, {r2, r3} + beq strcpy_unalign_return + b strcpy_unalign7 + +strcpy_unalign7_copy5bytes: + stmia r0!, {r2} + strb r3, [r0] +strcpy_unalign_return: + m_ret inst=pop + +strcpy_unalign7_copy6bytes: + stmia r0!, {r2} + strb r3, [r0], #1 + strb r4, [r0], #1 + m_ret inst=pop + +strcpy_unalign7_copy7bytes: + stmia r0!, {r2} + strb r3, [r0], #1 + strb r4, [r0], #1 + strb r5, [r0], #1 + m_ret inst=pop + + .p2align 2 + // Can read 6 bytes before possibly crossing a page. +strcpy_unalign6: + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + ldrb r4, [r1] + cbz r4, strcpy_unalign_copy5bytes + ldrb r5, [r1, #1] + cbz r5, strcpy_unalign_copy6bytes + + ldr r3, [r1], #4 + pld [r1, #64] + + tst r3, #0xff0000 + beq strcpy_unalign6_copy7bytes + lsrs ip, r3, #24 + stmia r0!, {r2, r3} + beq strcpy_unalign_return + b strcpy_unalign6 + +strcpy_unalign6_copy7bytes: + stmia r0!, {r2} + strh r3, [r0], #2 + lsr r3, #16 + strb r3, [r0] + m_ret inst=pop + + .p2align 2 + // Can read 5 bytes before possibly crossing a page. +strcpy_unalign5: + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + ldrb r4, [r1] + cbz r4, strcpy_unalign_copy5bytes + + ldr r3, [r1], #4 + + pld [r1, #64] + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + stmia r0!, {r2, r3} + b strcpy_unalign5 + +strcpy_unalign_copy5bytes: + stmia r0!, {r2} + strb r4, [r0] + m_ret inst=pop + +strcpy_unalign_copy6bytes: + stmia r0!, {r2} + strb r4, [r0], #1 + strb r5, [r0] + m_ret inst=pop + + .p2align 2 + // Can read 4 bytes before possibly crossing a page. +strcpy_unalign4: + ldmia r1!, {r2} + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + ldmia r1!, {r3} + pld [r1, #64] + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + stmia r0!, {r2, r3} + b strcpy_unalign4 + + .p2align 2 + // Can read 3 bytes before possibly crossing a page. +strcpy_unalign3: + ldrb r2, [r1] + cbz r2, strcpy_unalign3_copy1byte + ldrb r3, [r1, #1] + cbz r3, strcpy_unalign3_copy2bytes + ldrb r4, [r1, #2] + cbz r4, strcpy_unalign3_copy3bytes + + ldr r2, [r1], #4 + ldr r3, [r1], #4 + + pld [r1, #64] + + lsrs lr, r2, #24 + beq strcpy_unalign_copy4bytes + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + stmia r0!, {r2, r3} + b strcpy_unalign3 + +strcpy_unalign3_copy1byte: + strb r2, [r0] + m_ret inst=pop + +strcpy_unalign3_copy2bytes: + strb r2, [r0], #1 + strb r3, [r0] + m_ret inst=pop + +strcpy_unalign3_copy3bytes: + strb r2, [r0], #1 + strb r3, [r0], #1 + strb r4, [r0] + m_ret inst=pop + + .p2align 2 + // Can read 2 bytes before possibly crossing a page. +strcpy_unalign2: + ldrb r2, [r1] + cbz r2, strcpy_unalign_copy1byte + ldrb r3, [r1, #1] + cbz r3, strcpy_unalign_copy2bytes + + ldr r2, [r1], #4 + ldr r3, [r1], #4 + pld [r1, #64] + + tst r2, #0xff0000 + beq strcpy_unalign_copy3bytes + lsrs ip, r2, #24 + beq strcpy_unalign_copy4bytes + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + stmia r0!, {r2, r3} + b strcpy_unalign2 + + .p2align 2 + // Can read 1 byte before possibly crossing a page. +strcpy_unalign1: + ldrb r2, [r1] + cbz r2, strcpy_unalign_copy1byte + + ldr r2, [r1], #4 + ldr r3, [r1], #4 + + pld [r1, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + stmia r0!, {r2, r3} + b strcpy_unalign1 + +strcpy_unalign_copy1byte: + strb r2, [r0] + m_ret inst=pop + +strcpy_unalign_copy2bytes: + strb r2, [r0], #1 + strb r3, [r0] + m_ret inst=pop + +strcpy_unalign_copy3bytes: + strh r2, [r0], #2 + lsr r2, #16 + strb r2, [r0] + m_ret inst=pop + +strcpy_unalign_copy4bytes: + stmia r0, {r2} + m_ret inst=pop + +strcat_align_src: + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq strcat_align_to_32 + ldrb r2, [r0], #1 + cbz r2, strcat_r0_update + +strcat_align_to_32: + bcc strcat_align_to_64 + ldrb r2, [r0], #1 + cbz r2, strcat_r0_update + ldrb r2, [r0], #1 + cbz r2, strcat_r0_update + +strcat_align_to_64: + tst r3, #4 + beq strcat_mainloop + ldr r3, [r0], #4 + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcat_zero_in_second_register + b strcat_mainloop + +strcat_r0_update: + sub r0, r0, #1 + b strcat_r0_scan_done +END(strcat) diff --git a/libc/arch-arm/cortex-a9/bionic/strcpy.S b/libc/arch-arm/cortex-a9/bionic/strcpy.S new file mode 100644 index 000000000..9aa4f883d --- /dev/null +++ b/libc/arch-arm/cortex-a9/bionic/strcpy.S @@ -0,0 +1,456 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + + .syntax unified + + .thumb + .thumb_func + + .macro m_push + push {r0, r4, r5, lr} + .endm // m_push + + .macro m_ret inst + \inst {r0, r4, r5, pc} + .endm // m_ret + + .macro m_copy_byte reg, cmd, label + ldrb \reg, [r1], #1 + strb \reg, [r0], #1 + \cmd \reg, \label + .endm // m_copy_byte + +ENTRY(strcpy) + // Unroll the first 8 bytes that will be copied. + m_push + m_copy_byte reg=r2, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r3, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r4, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r5, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r2, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r3, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r4, cmd=cbz, label=strcpy_finish + m_copy_byte reg=r5, cmd=cbnz, label=strcpy_continue + +strcpy_finish: + m_ret inst=pop + +strcpy_continue: + pld [r1, #0] + ands r3, r0, #7 + bne strcpy_align_dst + +strcpy_check_src_align: + // At this point dst is aligned to a double word, check if src + // is also aligned to a double word. + ands r3, r1, #7 + bne strcpy_unaligned_copy + + .p2align 2 +strcpy_mainloop: + ldmia r1!, {r2, r3} + + pld [r1, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + stmia r0!, {r2, r3} + b strcpy_mainloop + +strcpy_zero_in_first_register: + lsls lr, ip, #17 + itt ne + strbne r2, [r0] + m_ret inst=popne + itt cs + strhcs r2, [r0] + m_ret inst=popcs + lsls ip, ip, #1 + itt eq + streq r2, [r0] + m_ret inst=popeq + strh r2, [r0], #2 + lsr r3, r2, #16 + strb r3, [r0] + m_ret inst=pop + +strcpy_zero_in_second_register: + lsls lr, ip, #17 + ittt ne + stmiane r0!, {r2} + strbne r3, [r0] + m_ret inst=popne + ittt cs + strcs r2, [r0], #4 + strhcs r3, [r0] + m_ret inst=popcs + lsls ip, ip, #1 + itt eq + stmiaeq r0, {r2, r3} + m_ret inst=popeq + stmia r0!, {r2} + strh r3, [r0], #2 + lsr r4, r3, #16 + strb r4, [r0] + m_ret inst=pop + +strcpy_align_dst: + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq strcpy_align_to_32 + + ldrb r2, [r1], #1 + strb r2, [r0], #1 + cbz r2, strcpy_complete + +strcpy_align_to_32: + bcc strcpy_align_to_64 + + ldrb r4, [r1], #1 + strb r4, [r0], #1 + cmp r4, #0 + it eq + m_ret inst=popeq + ldrb r5, [r1], #1 + strb r5, [r0], #1 + cmp r5, #0 + it eq + m_ret inst=popeq + +strcpy_align_to_64: + tst r3, #4 + beq strcpy_check_src_align + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + stmia r0!, {r2} + b strcpy_check_src_align + +strcpy_complete: + m_ret inst=pop + +strcpy_unaligned_copy: + // Dst is aligned to a double word, while src is at an unknown alignment. + // There are 7 different versions of the unaligned copy code + // to prevent overreading the src. The mainloop of every single version + // will store 64 bits per loop. The difference is how much of src can + // be read without potentially crossing a page boundary. + tbb [pc, r3] +strcpy_unaligned_branchtable: + .byte 0 + .byte ((strcpy_unalign7 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign6 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign5 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign4 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign3 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign2 - strcpy_unaligned_branchtable)/2) + .byte ((strcpy_unalign1 - strcpy_unaligned_branchtable)/2) + + .p2align 2 + // Can read 7 bytes before possibly crossing a page. +strcpy_unalign7: + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + ldrb r3, [r1] + cbz r3, strcpy_unalign7_copy5bytes + ldrb r4, [r1, #1] + cbz r4, strcpy_unalign7_copy6bytes + ldrb r5, [r1, #2] + cbz r5, strcpy_unalign7_copy7bytes + + ldr r3, [r1], #4 + pld [r1, #64] + + lsrs ip, r3, #24 + stmia r0!, {r2, r3} + beq strcpy_unalign_return + b strcpy_unalign7 + +strcpy_unalign7_copy5bytes: + stmia r0!, {r2} + strb r3, [r0] +strcpy_unalign_return: + m_ret inst=pop + +strcpy_unalign7_copy6bytes: + stmia r0!, {r2} + strb r3, [r0], #1 + strb r4, [r0], #1 + m_ret inst=pop + +strcpy_unalign7_copy7bytes: + stmia r0!, {r2} + strb r3, [r0], #1 + strb r4, [r0], #1 + strb r5, [r0], #1 + m_ret inst=pop + + .p2align 2 + // Can read 6 bytes before possibly crossing a page. +strcpy_unalign6: + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + ldrb r4, [r1] + cbz r4, strcpy_unalign_copy5bytes + ldrb r5, [r1, #1] + cbz r5, strcpy_unalign_copy6bytes + + ldr r3, [r1], #4 + pld [r1, #64] + + tst r3, #0xff0000 + beq strcpy_unalign6_copy7bytes + lsrs ip, r3, #24 + stmia r0!, {r2, r3} + beq strcpy_unalign_return + b strcpy_unalign6 + +strcpy_unalign6_copy7bytes: + stmia r0!, {r2} + strh r3, [r0], #2 + lsr r3, #16 + strb r3, [r0] + m_ret inst=pop + + .p2align 2 + // Can read 5 bytes before possibly crossing a page. +strcpy_unalign5: + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + ldrb r4, [r1] + cbz r4, strcpy_unalign_copy5bytes + + ldr r3, [r1], #4 + + pld [r1, #64] + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + stmia r0!, {r2, r3} + b strcpy_unalign5 + +strcpy_unalign_copy5bytes: + stmia r0!, {r2} + strb r4, [r0] + m_ret inst=pop + +strcpy_unalign_copy6bytes: + stmia r0!, {r2} + strb r4, [r0], #1 + strb r5, [r0] + m_ret inst=pop + + .p2align 2 + // Can read 4 bytes before possibly crossing a page. +strcpy_unalign4: + ldmia r1!, {r2} + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + ldmia r1!, {r3} + pld [r1, #64] + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + stmia r0!, {r2, r3} + b strcpy_unalign4 + + .p2align 2 + // Can read 3 bytes before possibly crossing a page. +strcpy_unalign3: + ldrb r2, [r1] + cbz r2, strcpy_unalign3_copy1byte + ldrb r3, [r1, #1] + cbz r3, strcpy_unalign3_copy2bytes + ldrb r4, [r1, #2] + cbz r4, strcpy_unalign3_copy3bytes + + ldr r2, [r1], #4 + ldr r3, [r1], #4 + + pld [r1, #64] + + lsrs lr, r2, #24 + beq strcpy_unalign_copy4bytes + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + stmia r0!, {r2, r3} + b strcpy_unalign3 + +strcpy_unalign3_copy1byte: + strb r2, [r0] + m_ret inst=pop + +strcpy_unalign3_copy2bytes: + strb r2, [r0], #1 + strb r3, [r0] + m_ret inst=pop + +strcpy_unalign3_copy3bytes: + strb r2, [r0], #1 + strb r3, [r0], #1 + strb r4, [r0] + m_ret inst=pop + + .p2align 2 + // Can read 2 bytes before possibly crossing a page. +strcpy_unalign2: + ldrb r2, [r1] + cbz r2, strcpy_unalign_copy1byte + ldrb r3, [r1, #1] + cbz r3, strcpy_unalign_copy2bytes + + ldr r2, [r1], #4 + ldr r3, [r1], #4 + pld [r1, #64] + + tst r2, #0xff0000 + beq strcpy_unalign_copy3bytes + lsrs ip, r2, #24 + beq strcpy_unalign_copy4bytes + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + stmia r0!, {r2, r3} + b strcpy_unalign2 + + .p2align 2 + // Can read 1 byte before possibly crossing a page. +strcpy_unalign1: + ldrb r2, [r1] + cbz r2, strcpy_unalign_copy1byte + + ldr r2, [r1], #4 + ldr r3, [r1], #4 + + pld [r1, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne strcpy_zero_in_second_register + + stmia r0!, {r2, r3} + b strcpy_unalign1 + +strcpy_unalign_copy1byte: + strb r2, [r0] + m_ret inst=pop + +strcpy_unalign_copy2bytes: + strb r2, [r0], #1 + strb r3, [r0] + m_ret inst=pop + +strcpy_unalign_copy3bytes: + strh r2, [r0], #2 + lsr r2, #16 + strb r2, [r0] + m_ret inst=pop + +strcpy_unalign_copy4bytes: + stmia r0, {r2} + m_ret inst=pop +END(strcpy) diff --git a/libc/arch-arm/cortex-a9/bionic/strlen.S b/libc/arch-arm/cortex-a9/bionic/strlen.S new file mode 100644 index 000000000..259eda0c4 --- /dev/null +++ b/libc/arch-arm/cortex-a9/bionic/strlen.S @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + + .syntax unified + + .thumb + .thumb_func + +ENTRY(strlen) + pld [r0, #0] + mov r1, r0 + + ands r3, r0, #7 + bne align_src + + .p2align 2 +mainloop: + ldmia r1!, {r2, r3} + + pld [r1, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne zero_in_second_register + b mainloop + +zero_in_first_register: + sub r0, r1, r0 + // Check for zero in byte 0. + lsls r2, ip, #17 + beq check_byte1_reg1 + + sub r0, r0, #8 + bx lr + +check_byte1_reg1: + bcc check_byte2_reg1 + + sub r0, r0, #7 + bx lr + +check_byte2_reg1: + // Check for zero in byte 2. + tst ip, #0x800000 + itt ne + subne r0, r0, #6 + bxne lr + sub r0, r0, #5 + bx lr + +zero_in_second_register: + sub r0, r1, r0 + // Check for zero in byte 0. + lsls r2, ip, #17 + beq check_byte1_reg2 + + sub r0, r0, #4 + bx lr + +check_byte1_reg2: + bcc check_byte2_reg2 + + sub r0, r0, #3 + bx lr + +check_byte2_reg2: + // Check for zero in byte 2. + tst ip, #0x800000 + itt ne + subne r0, r0, #2 + bxne lr + sub r0, r0, #1 + bx lr + +align_src: + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq align_to_32 + + ldrb r2, [r1], #1 + cbz r2, done + +align_to_32: + bcc align_to_64 + + ldrb r2, [r1], #1 + cbz r2, done + ldrb r2, [r1], #1 + cbz r2, done + +align_to_64: + tst r3, #4 + beq mainloop + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne zero_in_second_register + b mainloop + +done: + sub r0, r1, r0 + sub r0, r0, #1 + bx lr +END(strlen) diff --git a/libc/arch-arm/cortex-a9/cortex-a9.mk b/libc/arch-arm/cortex-a9/cortex-a9.mk index 5c684ed49..61a52c2ac 100644 --- a/libc/arch-arm/cortex-a9/cortex-a9.mk +++ b/libc/arch-arm/cortex-a9/cortex-a9.mk @@ -1,7 +1,8 @@ $(call libc-add-cpu-variant-src,MEMCPY,arch-arm/cortex-a9/bionic/memcpy.S) $(call libc-add-cpu-variant-src,MEMSET,arch-arm/cortex-a9/bionic/memset.S) +$(call libc-add-cpu-variant-src,STRCAT,arch-arm/cortex-a9/bionic/strcat.S) $(call libc-add-cpu-variant-src,STRCMP,arch-arm/cortex-a9/bionic/strcmp.S) -# Use cortex-a15 version of strlen. -$(call libc-add-cpu-variant-src,STRLEN,arch-arm/cortex-a15/bionic/strlen.S) +$(call libc-add-cpu-variant-src,STRCPY,arch-arm/cortex-a9/bionic/strcpy.S) +$(call libc-add-cpu-variant-src,STRLEN,arch-arm/cortex-a9/bionic/strlen.S) include bionic/libc/arch-arm/generic/generic.mk diff --git a/libc/arch-arm/bionic/strcpy.S b/libc/arch-arm/generic/bionic/strcpy.S similarity index 100% rename from libc/arch-arm/bionic/strcpy.S rename to libc/arch-arm/generic/bionic/strcpy.S diff --git a/libc/arch-arm/generic/generic.mk b/libc/arch-arm/generic/generic.mk index 18cad9da6..0b3f64494 100644 --- a/libc/arch-arm/generic/generic.mk +++ b/libc/arch-arm/generic/generic.mk @@ -1,4 +1,6 @@ $(call libc-add-cpu-variant-src,MEMCPY,arch-arm/generic/bionic/memcpy.S) $(call libc-add-cpu-variant-src,MEMSET,arch-arm/generic/bionic/memset.S) +$(call libc-add-cpu-variant-src,STRCAT,string/strcat.c) $(call libc-add-cpu-variant-src,STRCMP,arch-arm/generic/bionic/strcmp.S) +$(call libc-add-cpu-variant-src,STRCPY,arch-arm/generic/bionic/strcpy.c) $(call libc-add-cpu-variant-src,STRLEN,arch-arm/generic/bionic/strlen.c) diff --git a/libc/arch-arm/krait/krait.mk b/libc/arch-arm/krait/krait.mk index 288afbb61..1ff18e9bb 100644 --- a/libc/arch-arm/krait/krait.mk +++ b/libc/arch-arm/krait/krait.mk @@ -1,7 +1,9 @@ $(call libc-add-cpu-variant-src,MEMCPY,arch-arm/krait/bionic/memcpy.S) $(call libc-add-cpu-variant-src,MEMSET,arch-arm/krait/bionic/memset.S) $(call libc-add-cpu-variant-src,STRCMP,arch-arm/krait/bionic/strcmp.S) -# Use cortex-a15 version of strlen. +# Use cortex-a15 versions of strcat/strcpy/strlen. +$(call libc-add-cpu-variant-src,STRCAT,arch-arm/cortex-a15/bionic/strcat.S) +$(call libc-add-cpu-variant-src,STRCPY,arch-arm/cortex-a15/bionic/strcpy.S) $(call libc-add-cpu-variant-src,STRLEN,arch-arm/cortex-a15/bionic/strlen.S) include bionic/libc/arch-arm/generic/generic.mk From 1ce665416307628f4bcaced86faa64bdf9c489c3 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Mon, 5 Aug 2013 17:08:06 -0700 Subject: [PATCH 49/77] Fix strcpy.c that should have been strcpy.S. Change-Id: Ib4609baad3a14c8b0f37556269781fa2b06916dc --- libc/arch-arm/generic/generic.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/arch-arm/generic/generic.mk b/libc/arch-arm/generic/generic.mk index 0b3f64494..c3a5aa53b 100644 --- a/libc/arch-arm/generic/generic.mk +++ b/libc/arch-arm/generic/generic.mk @@ -2,5 +2,5 @@ $(call libc-add-cpu-variant-src,MEMCPY,arch-arm/generic/bionic/memcpy.S) $(call libc-add-cpu-variant-src,MEMSET,arch-arm/generic/bionic/memset.S) $(call libc-add-cpu-variant-src,STRCAT,string/strcat.c) $(call libc-add-cpu-variant-src,STRCMP,arch-arm/generic/bionic/strcmp.S) -$(call libc-add-cpu-variant-src,STRCPY,arch-arm/generic/bionic/strcpy.c) +$(call libc-add-cpu-variant-src,STRCPY,arch-arm/generic/bionic/strcpy.S) $(call libc-add-cpu-variant-src,STRLEN,arch-arm/generic/bionic/strlen.c) From 7c860db0747f6276a6e43984d43f8fa5181ea936 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Thu, 1 Aug 2013 13:13:33 -0700 Subject: [PATCH 50/77] Optimize __memset_chk, __memcpy_chk. This change creates assembler versions of __memcpy_chk/__memset_chk that is implemented in the memcpy/memset assembler code. This change avoids an extra call to memcpy/memset, instead allowing a simple fall through to occur from the chk code into the body of the real implementation. Testing: - Ran the libc_test on __memcpy_chk/__memset_chk on all nexus devices. - Wrote a small test executable that has three calls to __memcpy_chk and three calls to __memset_chk. First call dest_len is length + 1. Second call dest_len is length. Third call dest_len is length - 1. Verified that the first two calls pass, and the third fails. Examined the logcat output on all nexus devices to verify that the fortify error message was sent properly. - I benchmarked the new __memcpy_chk and __memset_chk on all systems. For __memcpy_chk and large copies, the savings is relatively small (about 1%). For small copies, the savings is large on cortex-a15/krait devices (between 5% to 30%). For cortex-a9 and small copies, the speed up is present, but relatively small (about 3% to 5%). For __memset_chk and large copies, the savings is also small (about 1%). However, all processors show larger speed-ups on small copies (about 30% to 100%). Bug: 9293744 Change-Id: I8926d59fe2673e36e8a27629e02a7b7059ebbc98 --- libc/Android.mk | 35 +++++++------- libc/arch-arm/arm.mk | 9 ++++ libc/arch-arm/cortex-a15/bionic/memcpy.S | 25 ++++++++++ libc/arch-arm/cortex-a15/bionic/memset.S | 29 +++++++++-- libc/arch-arm/cortex-a9/bionic/memcpy.S | 25 ++++++++++ libc/arch-arm/cortex-a9/bionic/memset.S | 24 ++++++++++ libc/arch-arm/generic/bionic/memcpy.S | 24 ++++++++++ libc/arch-arm/generic/bionic/memset.S | 24 ++++++++++ libc/arch-arm/krait/bionic/memcpy.S | 26 +++++++++- libc/arch-arm/krait/bionic/memset.S | 24 ++++++++++ .../__strrchr_chk.cpp} | 23 +++++---- libc/private/libc_events.h | 48 +++++++++++++++++++ libc/private/libc_logging.h | 14 +----- 13 files changed, 285 insertions(+), 45 deletions(-) rename libc/{string/__strrchr_chk.c => bionic/__strrchr_chk.cpp} (83%) create mode 100644 libc/private/libc_events.h diff --git a/libc/Android.mk b/libc/Android.mk index f353c416f..6ebb9e929 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -62,7 +62,6 @@ libc_common_src_files := \ string/strcspn.c \ string/strdup.c \ string/strpbrk.c \ - string/__strrchr_chk.c \ string/strsep.c \ string/strspn.c \ string/strstr.c \ @@ -181,6 +180,25 @@ libc_common_src_files := \ netbsd/nameser/ns_print.c \ netbsd/nameser/ns_samedomain.c \ +# Fortify implementations of libc functions. +libc_common_src_files += \ + bionic/__fgets_chk.cpp \ + bionic/__memcpy_chk.cpp \ + bionic/__memmove_chk.cpp \ + bionic/__memset_chk.cpp \ + bionic/__strcat_chk.cpp \ + bionic/__strchr_chk.cpp \ + bionic/__strcpy_chk.cpp \ + bionic/__strlcat_chk.cpp \ + bionic/__strlcpy_chk.cpp \ + bionic/__strlen_chk.cpp \ + bionic/__strncat_chk.cpp \ + bionic/__strncpy_chk.cpp \ + bionic/__strrchr_chk.cpp \ + bionic/__umask_chk.cpp \ + bionic/__vsnprintf_chk.cpp \ + bionic/__vsprintf_chk.cpp \ + libc_bionic_src_files := \ bionic/abort.cpp \ bionic/assert.cpp \ @@ -189,15 +207,11 @@ libc_bionic_src_files := \ bionic/__errno.c \ bionic/eventfd_read.cpp \ bionic/eventfd_write.cpp \ - bionic/__fgets_chk.cpp \ bionic/getauxval.cpp \ bionic/getcwd.cpp \ bionic/libc_init_common.cpp \ bionic/libc_logging.cpp \ bionic/libgen.cpp \ - bionic/__memcpy_chk.cpp \ - bionic/__memmove_chk.cpp \ - bionic/__memset_chk.cpp \ bionic/mmap.cpp \ bionic/pthread_attr.cpp \ bionic/pthread_detach.cpp \ @@ -220,24 +234,13 @@ libc_bionic_src_files := \ bionic/signalfd.cpp \ bionic/sigwait.cpp \ bionic/statvfs.cpp \ - bionic/__strcat_chk.cpp \ - bionic/__strchr_chk.cpp \ - bionic/__strcpy_chk.cpp \ bionic/strerror.cpp \ bionic/strerror_r.cpp \ - bionic/__strlcat_chk.cpp \ - bionic/__strlcpy_chk.cpp \ - bionic/__strlen_chk.cpp \ - bionic/__strncat_chk.cpp \ - bionic/__strncpy_chk.cpp \ bionic/strsignal.cpp \ bionic/stubs.cpp \ bionic/sysconf.cpp \ bionic/tdestroy.cpp \ bionic/tmpfile.cpp \ - bionic/__umask_chk.cpp \ - bionic/__vsnprintf_chk.cpp \ - bionic/__vsprintf_chk.cpp \ bionic/wait.cpp \ bionic/wchar.cpp \ diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index 7fb15a1c7..1d9863c32 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -26,6 +26,15 @@ _LIBC_ARCH_STATIC_SRC_FILES := \ _LIBC_ARCH_DYNAMIC_SRC_FILES := \ arch-arm/bionic/exidx_dynamic.c +# Remove the C++ fortify function implementations for which there is an +# arm assembler version. +_LIBC_FORTIFY_FILES_TO_REMOVE := \ + bionic/__memcpy_chk.cpp \ + bionic/__memset_chk.cpp \ + +libc_common_src_files := \ + $(filter-out $(_LIBC_FORTIFY_FILES_TO_REMOVE),$(libc_common_src_files)) + ifeq ($(strip $(wildcard bionic/libc/arch-arm/$(TARGET_CPU_VARIANT)/$(TARGET_CPU_VARIANT).mk)),) $(error "TARGET_CPU_VARIANT not set or set to an unknown value. Possible values are cortex-a7, cortex-a8, cortex-a9, cortex-a15, krait. Use generic for devices that do not have a CPU similar to any of the supported cpu variants.") endif diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy.S b/libc/arch-arm/cortex-a15/bionic/memcpy.S index d29706488..239402455 100644 --- a/libc/arch-arm/cortex-a15/bionic/memcpy.S +++ b/libc/arch-arm/cortex-a15/bionic/memcpy.S @@ -59,6 +59,7 @@ #include #include +#include "libc_events.h" .text .syntax unified @@ -66,6 +67,13 @@ #define CACHE_LINE_SIZE 64 +ENTRY(__memcpy_chk) + cmp r2, r3 + bgt fortify_check_failed + + // Fall through to memcpy... +END(__memcpy_chk) + ENTRY(memcpy) // Assumes that n >= 0, and dst, src are valid pointers. // For any sizes less than 832 use the neon code that doesn't @@ -321,4 +329,21 @@ dst_not_word_aligned: // Src is guaranteed to be at least word aligned by this point. b word_aligned + + + // Only reached when the __memcpy_chk check fails. +fortify_check_failed: + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+8) END(memcpy) + + .data +error_string: + .string "memcpy buffer overflow" diff --git a/libc/arch-arm/cortex-a15/bionic/memset.S b/libc/arch-arm/cortex-a15/bionic/memset.S index 2e1ad54c6..6c143ad81 100644 --- a/libc/arch-arm/cortex-a15/bionic/memset.S +++ b/libc/arch-arm/cortex-a15/bionic/memset.S @@ -28,19 +28,38 @@ #include #include +#include "libc_events.h" - /* - * Optimized memset() for ARM. + /* + * Optimized memset() for ARM. * * memset() returns its first argument. - */ + */ .fpu neon .syntax unified +ENTRY(__memset_chk) + cmp r2, r3 + bls done + + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMSET_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+8) + +END(__memset_chk) + ENTRY(bzero) mov r2, r1 mov r1, #0 + +done: // Fall through to memset... END(bzero) @@ -162,3 +181,7 @@ set_less_than_16_unknown_align: ldmfd sp!, {r0} bx lr END(memset) + + .data +error_string: + .string "memset buffer overflow" diff --git a/libc/arch-arm/cortex-a9/bionic/memcpy.S b/libc/arch-arm/cortex-a9/bionic/memcpy.S index 70e27b041..4e624d43a 100644 --- a/libc/arch-arm/cortex-a9/bionic/memcpy.S +++ b/libc/arch-arm/cortex-a9/bionic/memcpy.S @@ -28,6 +28,7 @@ #include #include +#include "libc_events.h" /* * This code assumes it is running on a processor that supports all arm v7 @@ -40,6 +41,13 @@ #define CACHE_LINE_SIZE 32 +ENTRY(__memcpy_chk) + cmp r2, r3 + bgt fortify_check_failed + + // Fall through to memcpy... +END(__memcpy_chk) + ENTRY(memcpy) .save {r0, lr} /* start preloading as early as possible */ @@ -208,4 +216,21 @@ ENTRY(memcpy) 6: ldmfd sp!, {r4, r5, r6, r7, r8} ldmfd sp!, {r0, pc} + + + // Only reached when the __memcpy_chk check fails. +fortify_check_failed: + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+8) END(memcpy) + + .data +error_string: + .string "memcpy buffer overflow" diff --git a/libc/arch-arm/cortex-a9/bionic/memset.S b/libc/arch-arm/cortex-a9/bionic/memset.S index b58aa456c..d01143039 100644 --- a/libc/arch-arm/cortex-a9/bionic/memset.S +++ b/libc/arch-arm/cortex-a9/bionic/memset.S @@ -28,6 +28,7 @@ #include #include +#include "libc_events.h" /* * This code assumes it is running on a processor that supports all arm v7 @@ -36,9 +37,28 @@ .fpu neon +ENTRY(__memset_chk) + cmp r2, r3 + bls done + + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMSET_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+8) + +END(__memset_chk) + ENTRY(bzero) mov r2, r1 mov r1, #0 + +done: + // Fall through to memset... END(bzero) /* memset() returns its first argument. */ @@ -150,3 +170,7 @@ ENTRY(memset) ldmfd sp!, {r0, r4-r7, lr} bx lr END(memset) + + .data +error_string: + .string "memset buffer overflow" diff --git a/libc/arch-arm/generic/bionic/memcpy.S b/libc/arch-arm/generic/bionic/memcpy.S index 6890a5502..24373d815 100644 --- a/libc/arch-arm/generic/bionic/memcpy.S +++ b/libc/arch-arm/generic/bionic/memcpy.S @@ -28,6 +28,7 @@ #include #include +#include "libc_events.h" /* * Optimized memcpy() for ARM. @@ -36,6 +37,13 @@ * so we have to preserve R0. */ +ENTRY(__memcpy_chk) + cmp r2, r3 + bgt fortify_check_failed + + // Fall through to memcpy... +END(__memcpy_chk) + ENTRY(memcpy) /* The stack must always be 64-bits aligned to be compliant with the * ARM ABI. Since we have to save R0, we might as well save R4 @@ -377,4 +385,20 @@ copy_last_3_and_return: add sp, sp, #28 ldmfd sp!, {r0, r4, lr} bx lr + + // Only reached when the __memcpy_chk check fails. +fortify_check_failed: + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+8) END(memcpy) + + .data +error_string: + .string "memcpy buffer overflow" diff --git a/libc/arch-arm/generic/bionic/memset.S b/libc/arch-arm/generic/bionic/memset.S index 3c034e015..399bae91e 100644 --- a/libc/arch-arm/generic/bionic/memset.S +++ b/libc/arch-arm/generic/bionic/memset.S @@ -27,6 +27,7 @@ */ #include +#include "libc_events.h" /* * Optimized memset() for ARM. @@ -34,9 +35,28 @@ * memset() returns its first argument. */ +ENTRY(__memset_chk) + cmp r2, r3 + bls done + + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMSET_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+8) + +END(__memset_chk) + ENTRY(bzero) mov r2, r1 mov r1, #0 + +done: + // Fall through to memset... END(bzero) ENTRY(memset) @@ -107,3 +127,7 @@ ENTRY(memset) ldmfd sp!, {r0, r4-r7, lr} bx lr END(memset) + + .data +error_string: + .string "memset buffer overflow" diff --git a/libc/arch-arm/krait/bionic/memcpy.S b/libc/arch-arm/krait/bionic/memcpy.S index 4a21709fb..3afe18ca8 100644 --- a/libc/arch-arm/krait/bionic/memcpy.S +++ b/libc/arch-arm/krait/bionic/memcpy.S @@ -30,6 +30,7 @@ #include #include +#include "libc_events.h" /* * This code assumes it is running on a processor that supports all arm v7 @@ -37,10 +38,17 @@ * cache line. */ +#define CACHE_LINE_SIZE 32 + .text .fpu neon -#define CACHE_LINE_SIZE 32 +ENTRY(__memcpy_chk) + cmp r2, r3 + bgt fortify_check_failed + + // Fall through to memcpy... +END(__memcpy_chk) ENTRY(memcpy) .save {r0, lr} @@ -124,4 +132,20 @@ ENTRY(memcpy) ldmfd sp!, {r0, lr} bx lr + + // Only reached when the __memcpy_chk check fails. +fortify_check_failed: + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+8) END(memcpy) + + .data +error_string: + .string "memcpy buffer overflow" diff --git a/libc/arch-arm/krait/bionic/memset.S b/libc/arch-arm/krait/bionic/memset.S index a2e2d80dc..4e4788b7b 100644 --- a/libc/arch-arm/krait/bionic/memset.S +++ b/libc/arch-arm/krait/bionic/memset.S @@ -28,6 +28,7 @@ #include #include +#include "libc_events.h" /* * This code assumes it is running on a processor that supports all arm v7 @@ -37,9 +38,28 @@ .fpu neon +ENTRY(__memset_chk) + cmp r2, r3 + bls done + + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_MEMSET_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+8) + +END(__memset_chk) + ENTRY(bzero) mov r2, r1 mov r1, #0 + +done: + // Fall through to memset... END(bzero) /* memset() returns its first argument. */ @@ -79,3 +99,7 @@ ENTRY(memset) ldmfd sp!, {r0} bx lr END(memset) + + .data +error_string: + .string "memset buffer overflow" diff --git a/libc/string/__strrchr_chk.c b/libc/bionic/__strrchr_chk.cpp similarity index 83% rename from libc/string/__strrchr_chk.c rename to libc/bionic/__strrchr_chk.cpp index c1e5d6605..14100f791 100644 --- a/libc/string/__strrchr_chk.c +++ b/libc/bionic/__strrchr_chk.cpp @@ -31,18 +31,17 @@ #include #include "libc_logging.h" -char * -__strrchr_chk(const char *p, int ch, size_t s_len) +extern "C" char* __strrchr_chk(const char *p, int ch, size_t s_len) { - char *save; + char *save; - for (save = NULL;; ++p, s_len--) { - if (s_len == 0) - __fortify_chk_fail("strrchr read beyond buffer", 0); - if (*p == (char) ch) - save = (char *)p; - if (!*p) - return(save); - } - /* NOTREACHED */ + for (save = NULL;; ++p, s_len--) { + if (s_len == 0) + __fortify_chk_fail("strrchr read beyond buffer", 0); + if (*p == (char) ch) + save = (char *)p; + if (!*p) + return(save); + } + /* NOTREACHED */ } diff --git a/libc/private/libc_events.h b/libc/private/libc_events.h new file mode 100644 index 000000000..5d20f4b81 --- /dev/null +++ b/libc/private/libc_events.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _LIBC_EVENTS_H +#define _LIBC_EVENTS_H + + +// This is going to be included in assembler code so only allow #define +// values instead of defining an enum. + +#define BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW 80100 +#define BIONIC_EVENT_STRCAT_BUFFER_OVERFLOW 80105 +#define BIONIC_EVENT_MEMMOVE_BUFFER_OVERFLOW 80110 +#define BIONIC_EVENT_STRNCAT_BUFFER_OVERFLOW 80115 +#define BIONIC_EVENT_STRNCPY_BUFFER_OVERFLOW 80120 +#define BIONIC_EVENT_MEMSET_BUFFER_OVERFLOW 80125 +#define BIONIC_EVENT_STRCPY_BUFFER_OVERFLOW 80130 + +#define BIONIC_EVENT_RESOLVER_OLD_RESPONSE 80300 +#define BIONIC_EVENT_RESOLVER_WRONG_SERVER 80305 +#define BIONIC_EVENT_RESOLVER_WRONG_QUERY 80310 + +#endif // _LIBC_EVENTS_H diff --git a/libc/private/libc_logging.h b/libc/private/libc_logging.h index f69e2ed5b..1cdcb6ece 100644 --- a/libc/private/libc_logging.h +++ b/libc/private/libc_logging.h @@ -36,19 +36,7 @@ __BEGIN_DECLS -enum { - BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW = 80100, - BIONIC_EVENT_STRCAT_BUFFER_OVERFLOW = 80105, - BIONIC_EVENT_MEMMOVE_BUFFER_OVERFLOW = 80110, - BIONIC_EVENT_STRNCAT_BUFFER_OVERFLOW = 80115, - BIONIC_EVENT_STRNCPY_BUFFER_OVERFLOW = 80120, - BIONIC_EVENT_MEMSET_BUFFER_OVERFLOW = 80125, - BIONIC_EVENT_STRCPY_BUFFER_OVERFLOW = 80130, - - BIONIC_EVENT_RESOLVER_OLD_RESPONSE = 80300, - BIONIC_EVENT_RESOLVER_WRONG_SERVER = 80305, - BIONIC_EVENT_RESOLVER_WRONG_QUERY = 80310, -}; +#include "libc_events.h" enum { ANDROID_LOG_UNKNOWN = 0, From 6771b9cef635f0da98932a1f0f877a07257ca088 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Wed, 7 Aug 2013 13:31:17 -0700 Subject: [PATCH 51/77] bionic: add function to name memory Only works on some kernels, and only on page-aligned regions of anonymous memory. It will show up in /proc/pid/maps as [anon:] and in /proc/pid/smaps as Name: Change-Id: If31667cf45ff41cc2a79a140ff68707526def80e --- libc/Android.mk | 1 + libc/bionic/name_mem.c | 53 ++++++++++++++++++++++++++++++++++ libc/private/bionic_name_mem.h | 40 +++++++++++++++++++++++++ 3 files changed, 94 insertions(+) create mode 100644 libc/bionic/name_mem.c create mode 100644 libc/private/bionic_name_mem.h diff --git a/libc/Android.mk b/libc/Android.mk index 6ebb9e929..6e089c568 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -104,6 +104,7 @@ libc_common_src_files := \ bionic/md5.c \ bionic/memmem.c \ bionic/memswap.c \ + bionic/name_mem.c \ bionic/openat.c \ bionic/open.c \ bionic/pathconf.c \ diff --git a/libc/bionic/name_mem.c b/libc/bionic/name_mem.c new file mode 100644 index 000000000..69e10c252 --- /dev/null +++ b/libc/bionic/name_mem.c @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "private/bionic_name_mem.h" + +/* + * Local definitions of custom prctl arguments to set a vma name in some kernels + */ +#define BIONIC_PR_SET_VMA 0x53564d41 +#define BIONIC_PR_SET_VMA_ANON_NAME 0 + +/* + * Names a region of memory. The name is expected to show up in /proc/pid/maps + * and /proc/pid/smaps. There is no guarantee that it will work, and it if it + * does work it is likely to only work on memory that was allocated with + * mmap(MAP_ANONYMOUS), and only on regions that are page aligned. name should + * be a pointer to a string that is valid for as long as the memory is mapped, + * preferably a compile-time constant string. + * + * Returns -1 on error and sets errno. If it returns an error naming page + * aligned anonymous memory the kernel doesn't support naming, and an alternate + * method of naming memory should be used (like ashmem). + */ +int __bionic_name_mem(void *addr, size_t len, const char *name) +{ + return prctl(BIONIC_PR_SET_VMA, BIONIC_PR_SET_VMA_ANON_NAME, + addr, len, name); +} diff --git a/libc/private/bionic_name_mem.h b/libc/private/bionic_name_mem.h new file mode 100644 index 000000000..9f6163d0b --- /dev/null +++ b/libc/private/bionic_name_mem.h @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#ifndef _BIONIC_NAME_MEM_H +#define _BIONIC_NAME_MEM_H + +#include +#include + +__BEGIN_DECLS + +int __bionic_name_mem(void *addr, size_t len, const char *name); + +__END_DECLS + +#endif From 7f4074d17d0e22e5e18e472c7e099490df8efaf2 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Wed, 7 Aug 2013 15:01:55 -0700 Subject: [PATCH 52/77] bionic: name malloc'd regions Use the new __bionic_name_mem function to name malloc'd memory as "libc_malloc" on kernels that support it. Change-Id: I7235eae6918fa107010039b9ab8b7cb362212272 --- libc/bionic/dlmalloc.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/libc/bionic/dlmalloc.c b/libc/bionic/dlmalloc.c index 78f2e1d56..66a825b33 100644 --- a/libc/bionic/dlmalloc.c +++ b/libc/bionic/dlmalloc.c @@ -16,6 +16,7 @@ #include "dlmalloc.h" +#include "private/bionic_name_mem.h" #include "private/libc_logging.h" // Send dlmalloc errors to the log. @@ -25,6 +26,11 @@ static void __bionic_heap_usage_error(const char* function, void* address); #define CORRUPTION_ERROR_ACTION(m) __bionic_heap_corruption_error(__FUNCTION__) #define USAGE_ERROR_ACTION(m,p) __bionic_heap_usage_error(__FUNCTION__, p) +/* Bionic named anonymous memory declarations */ +static void* named_anonymous_mmap(size_t length); +#define MMAP(s) named_anonymous_mmap(s) +#define DIRECT_MMAP(s) named_anonymous_mmap(s) + // Ugly inclusion of C file so that bionic specific #defines configure dlmalloc. #include "../upstream-dlmalloc/malloc.c" @@ -42,3 +48,15 @@ static void __bionic_heap_usage_error(const char* function, void* address) { // TODO: improve the debuggerd protocol so we can tell it to dump an address when we abort. *((int**) 0xdeadbaad) = (int*) address; } + +static void* named_anonymous_mmap(size_t length) +{ + void* ret; + ret = mmap(NULL, length, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (ret == MAP_FAILED) + return ret; + + __bionic_name_mem(ret, length, "libc_malloc"); + + return ret; +} From 840a114eb12773c5af39c0c97675b27aa6dee78c Mon Sep 17 00:00:00 2001 From: Elliott Hughes Date: Thu, 8 Aug 2013 17:13:33 -0700 Subject: [PATCH 53/77] Add futimens. (cherry picked from commit d0be7c8f9a06b3ca8ea7647ea35c8f9dc63f0fe1) Bug: 10239370 Change-Id: I0087e85a94d83b6ce68ec6a0768c44cbe4bd0132 --- libc/Android.mk | 1 + libc/bionic/futimens.cpp | 34 ++++++++++++++++++++++++++ libc/include/sys/stat.h | 3 ++- tests/Android.mk | 1 + tests/sys_stat_test.cpp | 53 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 libc/bionic/futimens.cpp create mode 100644 tests/sys_stat_test.cpp diff --git a/libc/Android.mk b/libc/Android.mk index 6e089c568..a6cbf6162 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -208,6 +208,7 @@ libc_bionic_src_files := \ bionic/__errno.c \ bionic/eventfd_read.cpp \ bionic/eventfd_write.cpp \ + bionic/futimens.cpp \ bionic/getauxval.cpp \ bionic/getcwd.cpp \ bionic/libc_init_common.cpp \ diff --git a/libc/bionic/futimens.cpp b/libc/bionic/futimens.cpp new file mode 100644 index 000000000..1ca8eb59a --- /dev/null +++ b/libc/bionic/futimens.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include + +int futimens(int fd, const struct timespec times[2]) { + return utimensat(fd, NULL, times, 0); +} diff --git a/libc/include/sys/stat.h b/libc/include/sys/stat.h index c7715a3ce..10627b953 100644 --- a/libc/include/sys/stat.h +++ b/libc/include/sys/stat.h @@ -168,7 +168,8 @@ extern int renameat(int olddirfd, const char *oldpath, int newdirfd, const char # define UTIME_NOW ((1l << 30) - 1l) # define UTIME_OMIT ((1l << 30) - 2l) -extern int utimensat (int fd, const char *path, const struct timespec times[2], int flags); +extern int utimensat(int fd, const char *path, const struct timespec times[2], int flags); +extern int futimens(int fd, const struct timespec times[2]); __END_DECLS diff --git a/tests/Android.mk b/tests/Android.mk index efee17bca..177e4520b 100644 --- a/tests/Android.mk +++ b/tests/Android.mk @@ -80,6 +80,7 @@ test_src_files = \ string_test.cpp \ strings_test.cpp \ stubs_test.cpp \ + sys_stat_test.cpp \ system_properties_test.cpp \ time_test.cpp \ unistd_test.cpp \ diff --git a/tests/sys_stat_test.cpp b/tests/sys_stat_test.cpp new file mode 100644 index 000000000..a23100a6c --- /dev/null +++ b/tests/sys_stat_test.cpp @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include + +TEST(sys_stat, futimens) { + FILE* fp = tmpfile(); + ASSERT_TRUE(fp != NULL); + + int fd = fileno(fp); + ASSERT_NE(fd, -1); + + timespec times[2]; + times[0].tv_sec = 123; + times[0].tv_nsec = 0; + times[1].tv_sec = 456; + times[1].tv_nsec = 0; + ASSERT_EQ(0, futimens(fd, times)) << strerror(errno); + + struct stat sb; + ASSERT_EQ(0, fstat(fd, &sb)); + ASSERT_EQ(times[0].tv_sec, static_cast(sb.st_atime)); + ASSERT_EQ(times[1].tv_sec, static_cast(sb.st_mtime)); + + fclose(fp); +} + +TEST(sys_stat, futimens_EBADF) { + timespec times[2]; + times[0].tv_sec = 123; + times[0].tv_nsec = 0; + times[1].tv_sec = 456; + times[1].tv_nsec = 0; + ASSERT_EQ(-1, futimens(-1, times)); + ASSERT_EQ(EBADF, errno); +} From 5951e84646553cdcba0c619f1424f2d934eeabb2 Mon Sep 17 00:00:00 2001 From: Elliott Hughes Date: Mon, 12 Aug 2013 12:07:05 -0700 Subject: [PATCH 54/77] Fix our missing abs/labs/llabs/imaxabs (and imaxdiv) symbols. (cherry picked from commit aec2ffbc5a7b45368ceab3663265fbb659968fe7) Change-Id: Ib61b52dc37e63493094a0a7af23bf89d4a9f0ce5 --- libc/Android.mk | 5 +++ libc/include/inttypes.h | 7 ++- libc/include/stdlib.h | 14 ++---- libc/upstream-freebsd/lib/libc/stdlib/abs.c | 43 ++++++++++++++++++ .../lib/libc/stdlib/imaxabs.c | 36 +++++++++++++++ .../lib/libc/stdlib/imaxdiv.c | 45 +++++++++++++++++++ libc/upstream-freebsd/lib/libc/stdlib/labs.c | 43 ++++++++++++++++++ libc/upstream-freebsd/lib/libc/stdlib/llabs.c | 36 +++++++++++++++ 8 files changed, 216 insertions(+), 13 deletions(-) create mode 100644 libc/upstream-freebsd/lib/libc/stdlib/abs.c create mode 100644 libc/upstream-freebsd/lib/libc/stdlib/imaxabs.c create mode 100644 libc/upstream-freebsd/lib/libc/stdlib/imaxdiv.c create mode 100644 libc/upstream-freebsd/lib/libc/stdlib/labs.c create mode 100644 libc/upstream-freebsd/lib/libc/stdlib/llabs.c diff --git a/libc/Android.mk b/libc/Android.mk index a6cbf6162..9610c1400 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -287,7 +287,12 @@ libc_upstream_freebsd_src_files := \ upstream-freebsd/lib/libc/stdio/tempnam.c \ upstream-freebsd/lib/libc/stdio/tmpnam.c \ upstream-freebsd/lib/libc/stdio/wsetup.c \ + upstream-freebsd/lib/libc/stdlib/abs.c \ upstream-freebsd/lib/libc/stdlib/getopt_long.c \ + upstream-freebsd/lib/libc/stdlib/imaxabs.c \ + upstream-freebsd/lib/libc/stdlib/imaxdiv.c \ + upstream-freebsd/lib/libc/stdlib/labs.c \ + upstream-freebsd/lib/libc/stdlib/llabs.c \ upstream-freebsd/lib/libc/stdlib/qsort.c \ upstream-freebsd/lib/libc/stdlib/realpath.c \ upstream-freebsd/lib/libc/string/wcpcpy.c \ diff --git a/libc/include/inttypes.h b/libc/include/inttypes.h index 73b22db44..760670fca 100644 --- a/libc/include/inttypes.h +++ b/libc/include/inttypes.h @@ -249,13 +249,16 @@ typedef struct { } imaxdiv_t; __BEGIN_DECLS -intmax_t imaxabs(intmax_t); -imaxdiv_t imaxdiv(intmax_t, intmax_t); + +intmax_t imaxabs(intmax_t) __pure2; +imaxdiv_t imaxdiv(intmax_t, intmax_t) __pure2; + intmax_t strtoimax(const char *, char **, int); uintmax_t strtoumax(const char *, char **, int); intmax_t strntoimax(const char *nptr, char **endptr, int base, size_t n); uintmax_t strntoumax(const char *nptr, char **endptr, int base, size_t n); + __END_DECLS #endif /* _INTTYPES_H_ */ diff --git a/libc/include/stdlib.h b/libc/include/stdlib.h index e728cb674..9fa84c1d3 100644 --- a/libc/include/stdlib.h +++ b/libc/include/stdlib.h @@ -77,17 +77,9 @@ static __inline__ double atof(const char *nptr) return (strtod(nptr, NULL)); } -static __inline__ int abs(int __n) { - return (__n < 0) ? -__n : __n; -} - -static __inline__ long labs(long __n) { - return (__n < 0L) ? -__n : __n; -} - -static __inline__ long long llabs(long long __n) { - return (__n < 0LL) ? -__n : __n; -} +extern int abs(int) __pure2; +extern long labs(long) __pure2; +extern long long llabs(long long) __pure2; extern char * realpath(const char *path, char *resolved); extern int system(const char * string); diff --git a/libc/upstream-freebsd/lib/libc/stdlib/abs.c b/libc/upstream-freebsd/lib/libc/stdlib/abs.c new file mode 100644 index 000000000..87589474e --- /dev/null +++ b/libc/upstream-freebsd/lib/libc/stdlib/abs.c @@ -0,0 +1,43 @@ +/*- + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(LIBC_SCCS) && !defined(lint) +static char sccsid[] = "@(#)abs.c 8.1 (Berkeley) 6/4/93"; +#endif /* LIBC_SCCS and not lint */ +#include +__FBSDID("$FreeBSD$"); + +#include + +int +abs(j) + int j; +{ + return(j < 0 ? -j : j); +} diff --git a/libc/upstream-freebsd/lib/libc/stdlib/imaxabs.c b/libc/upstream-freebsd/lib/libc/stdlib/imaxabs.c new file mode 100644 index 000000000..35e3dee79 --- /dev/null +++ b/libc/upstream-freebsd/lib/libc/stdlib/imaxabs.c @@ -0,0 +1,36 @@ +/*- + * Copyright (c) 2001 Mike Barcroft + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include + +intmax_t +imaxabs(intmax_t j) +{ + return (j < 0 ? -j : j); +} diff --git a/libc/upstream-freebsd/lib/libc/stdlib/imaxdiv.c b/libc/upstream-freebsd/lib/libc/stdlib/imaxdiv.c new file mode 100644 index 000000000..7dae4675e --- /dev/null +++ b/libc/upstream-freebsd/lib/libc/stdlib/imaxdiv.c @@ -0,0 +1,45 @@ +/*- + * Copyright (c) 2001 Mike Barcroft + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include + +/* See comments in div.c for implementation details. */ +imaxdiv_t +imaxdiv(intmax_t numer, intmax_t denom) +{ + imaxdiv_t retval; + + retval.quot = numer / denom; + retval.rem = numer % denom; + if (numer >= 0 && retval.rem < 0) { + retval.quot++; + retval.rem -= denom; + } + return (retval); +} diff --git a/libc/upstream-freebsd/lib/libc/stdlib/labs.c b/libc/upstream-freebsd/lib/libc/stdlib/labs.c new file mode 100644 index 000000000..816370eee --- /dev/null +++ b/libc/upstream-freebsd/lib/libc/stdlib/labs.c @@ -0,0 +1,43 @@ +/*- + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if defined(LIBC_SCCS) && !defined(lint) +static char sccsid[] = "@(#)labs.c 8.1 (Berkeley) 6/4/93"; +#endif /* LIBC_SCCS and not lint */ +#include +__FBSDID("$FreeBSD$"); + +#include + +long +labs(j) + long j; +{ + return(j < 0 ? -j : j); +} diff --git a/libc/upstream-freebsd/lib/libc/stdlib/llabs.c b/libc/upstream-freebsd/lib/libc/stdlib/llabs.c new file mode 100644 index 000000000..2bfbada80 --- /dev/null +++ b/libc/upstream-freebsd/lib/libc/stdlib/llabs.c @@ -0,0 +1,36 @@ +/*- + * Copyright (c) 2001 Mike Barcroft + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include + +long long +llabs(long long j) +{ + return (j < 0 ? -j : j); +} From 3912f90e56780ef54c68c796ff6793f5692e7658 Mon Sep 17 00:00:00 2001 From: Elliott Hughes Date: Tue, 13 Aug 2013 14:30:59 -0700 Subject: [PATCH 55/77] Make ctype.h a little less unhygienic. This caused trouble for stlport. (cherry picked from commit 5e9b20f7212c23110693a4bd1f122ccac3fece80) Change-Id: I85d09c12c6de1e8502f83e0ef0f17747fdfe08a5 --- libc/include/ctype.h | 38 +++++++++++++++++++------------------- libc/stdlib/ctype_.c | 12 +++++++++++- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/libc/include/ctype.h b/libc/include/ctype.h index 58b76eae6..5557e3181 100644 --- a/libc/include/ctype.h +++ b/libc/include/ctype.h @@ -42,14 +42,14 @@ #include -#define _U 0x01 -#define _L 0x02 -#define _N 0x04 -#define _S 0x08 -#define _P 0x10 -#define _C 0x20 -#define _X 0x40 -#define _B 0x80 +#define _CTYPE_U 0x01 +#define _CTYPE_L 0x02 +#define _CTYPE_N 0x04 +#define _CTYPE_S 0x08 +#define _CTYPE_P 0x10 +#define _CTYPE_C 0x20 +#define _CTYPE_X 0x40 +#define _CTYPE_B 0x80 __BEGIN_DECLS @@ -101,57 +101,57 @@ int _toupper(int); __CTYPE_INLINE int isalnum(int c) { - return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & (_U|_L|_N))); + return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & (_CTYPE_U|_CTYPE_L|_CTYPE_N))); } __CTYPE_INLINE int isalpha(int c) { - return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & (_U|_L))); + return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & (_CTYPE_U|_CTYPE_L))); } __CTYPE_INLINE int iscntrl(int c) { - return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & _C)); + return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & _CTYPE_C)); } __CTYPE_INLINE int isdigit(int c) { - return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & _N)); + return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & _CTYPE_N)); } __CTYPE_INLINE int isgraph(int c) { - return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & (_P|_U|_L|_N))); + return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & (_CTYPE_P|_CTYPE_U|_CTYPE_L|_CTYPE_N))); } __CTYPE_INLINE int islower(int c) { - return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & _L)); + return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & _CTYPE_L)); } __CTYPE_INLINE int isprint(int c) { - return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & (_P|_U|_L|_N|_B))); + return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & (_CTYPE_P|_CTYPE_U|_CTYPE_L|_CTYPE_N|_CTYPE_B))); } __CTYPE_INLINE int ispunct(int c) { - return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & _P)); + return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & _CTYPE_P)); } __CTYPE_INLINE int isspace(int c) { - return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & _S)); + return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & _CTYPE_S)); } __CTYPE_INLINE int isupper(int c) { - return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & _U)); + return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & _CTYPE_U)); } __CTYPE_INLINE int isxdigit(int c) { - return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & (_N|_X))); + return (c == -1 ? 0 : ((_ctype_ + 1)[(unsigned char)c] & (_CTYPE_N|_CTYPE_X))); } __CTYPE_INLINE int tolower(int c) diff --git a/libc/stdlib/ctype_.c b/libc/stdlib/ctype_.c index cf32f1686..3703f64b7 100644 --- a/libc/stdlib/ctype_.c +++ b/libc/stdlib/ctype_.c @@ -36,6 +36,15 @@ #include #include "ctype_private.h" +#define _U _CTYPE_U +#define _L _CTYPE_L +#define _N _CTYPE_N +#define _S _CTYPE_S +#define _P _CTYPE_P +#define _C _CTYPE_C +#define _X _CTYPE_X +#define _B _CTYPE_B + const char _C_ctype_[1 + CTYPE_NUM_CHARS] = { 0, _C, _C, _C, _C, _C, _C, _C, _C, @@ -77,6 +86,8 @@ const char _C_ctype_[1 + CTYPE_NUM_CHARS] = { const char *_ctype_ = _C_ctype_; +// TODO: fix the header file so we don't have to duplicate all this inlined stuff. + #if 1 /* ndef NDEBUG */ int isalnum(int c) { @@ -155,4 +166,3 @@ int toascii(int c) #endif /* __BSD_VISIBLE || __XPG_VISIBLE */ #endif /* !NDBEUG */ - From f0c3d909136167fdbe32b7815e5e1e02b4c35d62 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Wed, 7 Aug 2013 13:09:51 -0700 Subject: [PATCH 56/77] Create optimized __strcpy_chk/__strcat_chk. This change pulls the memcpy code out into a new file so that the __strcpy_chk and __strcat_chk can use it with an include. The new versions of the two chk functions uses assembly versions of strlen and memcpy to implement this check. This allows near parity with the assembly versions of strcpy/strcat. It also means that as memcpy implementations get faster, so do the chk functions. Other included changes: - Change all of the assembly labels to local labels. The other labels confuse gdb and mess up backtracing. - Add .cfi_startproc and .cfi_endproc directives so that gdb is not confused when falling through from one function to another. - Change all functions to use cfi directives since they are more powerful. - Move the memcpy_chk fail code outside of the memcpy function definition so that backtraces work properly. - Preserve lr before the calls to __fortify_chk_fail so that the backtrace actually works. Testing: - Ran the bionic unit tests. Verified all error messages in logs are set correctly. - Ran libc_test, replacing strcpy with __strcpy_chk and replacing strcat with __strcat_chk. - Ran the debugger on nexus10, nexus4, and old nexus7. Verified that the backtrace is correct for all fortify check failures. Also verify that when falling through from __memcpy_chk to memcpy that the backtrace is still correct. Also verified the same for __memset_chk and bzero. Verified the two different paths in the cortex-a9 memset routine that save variables to the stack still show the backtrace properly. Bug: 9293744 Change-Id: Id5aec8c3cb14101d91bd125eaf3770c9c8aa3f57 (cherry picked from commit 2be91915dcecc956d14ff281db0c7d216ca98af2) --- libc/arch-arm/arm.mk | 2 + .../arch-arm/cortex-a15/bionic/__strcat_chk.S | 215 +++++++++++++ .../arch-arm/cortex-a15/bionic/__strcpy_chk.S | 176 ++++++++++ libc/arch-arm/cortex-a15/bionic/memcpy.S | 281 ++-------------- libc/arch-arm/cortex-a15/bionic/memcpy_base.S | 303 ++++++++++++++++++ libc/arch-arm/cortex-a15/bionic/memset.S | 55 ++-- libc/arch-arm/cortex-a15/cortex-a15.mk | 2 + libc/arch-arm/cortex-a9/bionic/__strcat_chk.S | 218 +++++++++++++ libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S | 178 ++++++++++ libc/arch-arm/cortex-a9/bionic/memcpy.S | 199 ++---------- libc/arch-arm/cortex-a9/bionic/memcpy_base.S | 206 ++++++++++++ libc/arch-arm/cortex-a9/bionic/memset.S | 27 +- libc/arch-arm/cortex-a9/cortex-a9.mk | 2 + libc/arch-arm/generic/generic.mk | 2 + libc/arch-arm/krait/bionic/__strcat_chk.S | 215 +++++++++++++ libc/arch-arm/krait/bionic/__strcpy_chk.S | 175 ++++++++++ libc/arch-arm/krait/bionic/memcpy.S | 110 ++----- libc/arch-arm/krait/bionic/memcpy_base.S | 117 +++++++ libc/arch-arm/krait/bionic/memset.S | 18 +- libc/arch-arm/krait/krait.mk | 2 + 20 files changed, 1950 insertions(+), 553 deletions(-) create mode 100644 libc/arch-arm/cortex-a15/bionic/__strcat_chk.S create mode 100644 libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S create mode 100644 libc/arch-arm/cortex-a15/bionic/memcpy_base.S create mode 100644 libc/arch-arm/cortex-a9/bionic/__strcat_chk.S create mode 100644 libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S create mode 100644 libc/arch-arm/cortex-a9/bionic/memcpy_base.S create mode 100644 libc/arch-arm/krait/bionic/__strcat_chk.S create mode 100644 libc/arch-arm/krait/bionic/__strcpy_chk.S create mode 100644 libc/arch-arm/krait/bionic/memcpy_base.S diff --git a/libc/arch-arm/arm.mk b/libc/arch-arm/arm.mk index 1d9863c32..e87ef3803 100644 --- a/libc/arch-arm/arm.mk +++ b/libc/arch-arm/arm.mk @@ -31,6 +31,8 @@ _LIBC_ARCH_DYNAMIC_SRC_FILES := \ _LIBC_FORTIFY_FILES_TO_REMOVE := \ bionic/__memcpy_chk.cpp \ bionic/__memset_chk.cpp \ + bionic/__strcpy_chk.cpp \ + bionic/__strcat_chk.cpp \ libc_common_src_files := \ $(filter-out $(_LIBC_FORTIFY_FILES_TO_REMOVE),$(libc_common_src_files)) diff --git a/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S b/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S new file mode 100644 index 000000000..08dc78a9a --- /dev/null +++ b/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S @@ -0,0 +1,215 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include "libc_events.h" + + .syntax unified + + .thumb + .thumb_func + +// Get the length of src string, then get the source of the dst string. +// Check that the two lengths together don't exceed the threshold, then +// do a memcpy of the data. +ENTRY(__strcat_chk) + .cfi_startproc + pld [r0, #0] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + push {r4, r5} + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 0 + + mov lr, r2 + + // Save the dst register to r5 + mov r5, r0 + + // Zero out r4 + eor r4, r4, r4 + + // r1 contains the address of the string to count. +.L_strlen_start: + mov r0, r1 + ands r3, r1, #7 + beq .L_mainloop + + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq .L_align_to_32 + + ldrb r2, [r1], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_32: + bcc .L_align_to_64 + ands ip, r3, #2 + beq .L_align_to_64 + + ldrb r2, [r1], #1 + cbz r2, .L_update_count_and_finish + ldrb r2, [r1], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_64: + tst r3, #4 + beq .L_mainloop + ldr r3, [r1], #4 + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + + .p2align 2 +.L_mainloop: + ldrd r2, r3, [r1], #8 + + pld [r1, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne .L_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + b .L_mainloop + +.L_update_count_and_finish: + sub r3, r1, r0 + sub r3, r3, #1 + b .L_finish + +.L_zero_in_first_register: + sub r3, r1, r0 + lsls r2, ip, #17 + bne .L_sub8_and_finish + bcs .L_sub7_and_finish + lsls ip, ip, #1 + bne .L_sub6_and_finish + + sub r3, r3, #5 + b .L_finish + +.L_sub8_and_finish: + sub r3, r3, #8 + b .L_finish + +.L_sub7_and_finish: + sub r3, r3, #7 + b .L_finish + +.L_sub6_and_finish: + sub r3, r3, #6 + b .L_finish + +.L_zero_in_second_register: + sub r3, r1, r0 + lsls r2, ip, #17 + bne .L_sub4_and_finish + bcs .L_sub3_and_finish + lsls ip, ip, #1 + bne .L_sub2_and_finish + + sub r3, r3, #1 + b .L_finish + +.L_sub4_and_finish: + sub r3, r3, #4 + b .L_finish + +.L_sub3_and_finish: + sub r3, r3, #3 + b .L_finish + +.L_sub2_and_finish: + sub r3, r3, #2 + +.L_finish: + cmp r4, #0 + bne .L_strlen_done + + // Time to get the dst string length. + mov r1, r5 + + // Save the original source address to r5. + mov r5, r0 + + // Save the current length (adding 1 for the terminator). + add r4, r3, #1 + b .L_strlen_start + + // r0 holds the pointer to the dst string. + // r3 holds the dst string length. + // r4 holds the src string length + 1. +.L_strlen_done: + add r2, r3, r4 + cmp r2, lr + bgt .L_fortify_check_failed + + // Set up the registers for the memcpy code. + mov r1, r5 + pld [r1, #64] + mov r2, r4 + add r0, r0, r3 + pop {r4, r5} + .cfi_adjust_cfa_offset -8 + .cfi_restore r4 + .cfi_restore r5 + + #include "memcpy_base.S" + +.L_fortify_check_failed: + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 + + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_STRCAT_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+4) + + .cfi_endproc +END(__strcat_chk) + + .data +error_string: + .string "strcat buffer overflow" diff --git a/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S b/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S new file mode 100644 index 000000000..9fde59015 --- /dev/null +++ b/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S @@ -0,0 +1,176 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include "libc_events.h" + + .syntax unified + + .thumb + .thumb_func + +// Get the length of the source string first, then do a memcpy of the data +// instead of a strcpy. +ENTRY(__strcpy_chk) + .cfi_startproc + pld [r0, #0] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + + mov lr, r2 + mov r0, r1 + + ands r3, r1, #7 + beq .L_mainloop + + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq .L_align_to_32 + + ldrb r2, [r0], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_32: + bcc .L_align_to_64 + ands ip, r3, #2 + beq .L_align_to_64 + + ldrb r2, [r0], #1 + cbz r2, .L_update_count_and_finish + ldrb r2, [r0], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_64: + tst r3, #4 + beq .L_mainloop + ldr r3, [r0], #4 + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + + .p2align 2 +.L_mainloop: + ldrd r2, r3, [r0], #8 + + pld [r0, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne .L_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + b .L_mainloop + +.L_update_count_and_finish: + sub r3, r0, r1 + sub r3, r3, #1 + b .L_check_size + +.L_zero_in_first_register: + sub r3, r0, r1 + lsls r2, ip, #17 + bne .L_sub8_and_finish + bcs .L_sub7_and_finish + lsls ip, ip, #1 + bne .L_sub6_and_finish + + sub r3, r3, #5 + b .L_check_size + +.L_sub8_and_finish: + sub r3, r3, #8 + b .L_check_size + +.L_sub7_and_finish: + sub r3, r3, #7 + b .L_check_size + +.L_sub6_and_finish: + sub r3, r3, #6 + b .L_check_size + +.L_zero_in_second_register: + sub r3, r0, r1 + lsls r2, ip, #17 + bne .L_sub4_and_finish + bcs .L_sub3_and_finish + lsls ip, ip, #1 + bne .L_sub2_and_finish + + sub r3, r3, #1 + b .L_check_size + +.L_sub4_and_finish: + sub r3, r3, #4 + b .L_check_size + +.L_sub3_and_finish: + sub r3, r3, #3 + b .L_check_size + +.L_sub2_and_finish: + sub r3, r3, #2 + +.L_check_size: + pld [r1, #0] + pld [r1, #64] + ldr r0, [sp] + cmp r3, lr + bge .L_fortify_check_failed + + // Add 1 for copy length to get the string terminator. + add r2, r3, #1 + + #include "memcpy_base.S" + +.L_fortify_check_failed: + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_STRCPY_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+4) + + .cfi_endproc +END(__strcpy_chk) + + .data +error_string: + .string "strcpy buffer overflow" diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy.S b/libc/arch-arm/cortex-a15/bionic/memcpy.S index 239402455..8052d62ef 100644 --- a/libc/arch-arm/cortex-a15/bionic/memcpy.S +++ b/libc/arch-arm/cortex-a15/bionic/memcpy.S @@ -53,11 +53,8 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ - /* Prototype: void *memcpy (void *dst, const void *src, size_t count). */ +// Prototype: void *memcpy (void *dst, const void *src, size_t count). - // This version is tuned for the Cortex-A15 processor. - -#include #include #include "libc_events.h" @@ -65,274 +62,34 @@ .syntax unified .fpu neon -#define CACHE_LINE_SIZE 64 - ENTRY(__memcpy_chk) + .cfi_startproc cmp r2, r3 - bgt fortify_check_failed + bgt __memcpy_chk_fail // Fall through to memcpy... + .cfi_endproc END(__memcpy_chk) ENTRY(memcpy) - // Assumes that n >= 0, and dst, src are valid pointers. - // For any sizes less than 832 use the neon code that doesn't - // care about the src alignment. This avoids any checks - // for src alignment, and offers the best improvement since - // smaller sized copies are dominated by the overhead of - // the pre and post main loop. - // For larger copies, if src and dst cannot both be aligned to - // word boundaries, use the neon code. - // For all other copies, align dst to a double word boundary - // and copy using LDRD/STRD instructions. - - // Save registers (r0 holds the return value): - // optimized push {r0, lr}. - .save {r0, lr} - pld [r1, #(CACHE_LINE_SIZE*16)] + .cfi_startproc + pld [r1, #64] push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 - cmp r2, #16 - blo copy_less_than_16_unknown_align + #include "memcpy_base.S" + .cfi_endproc +END(memcpy) - cmp r2, #832 - bge check_alignment + .cfi_startproc +__memcpy_chk_fail: + // Preserve lr for backtrace. + push {lr} + .cfi_def_cfa_offset 4 + .cfi_rel_offset lr, 0 -copy_unknown_alignment: - // Unknown alignment of src and dst. - // Assumes that the first few bytes have already been prefetched. - - // Align destination to 128 bits. The mainloop store instructions - // require this alignment or they will throw an exception. - rsb r3, r0, #0 - ands r3, r3, #0xF - beq 2f - - // Copy up to 15 bytes (count in r3). - sub r2, r2, r3 - movs ip, r3, lsl #31 - - itt mi - ldrbmi lr, [r1], #1 - strbmi lr, [r0], #1 - itttt cs - ldrbcs ip, [r1], #1 - ldrbcs lr, [r1], #1 - strbcs ip, [r0], #1 - strbcs lr, [r0], #1 - - movs ip, r3, lsl #29 - bge 1f - // Copies 4 bytes, dst 32 bits aligned before, at least 64 bits after. - vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! - vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]! -1: bcc 2f - // Copies 8 bytes, dst 64 bits aligned before, at least 128 bits after. - vld1.8 {d0}, [r1]! - vst1.8 {d0}, [r0, :64]! - -2: // Make sure we have at least 64 bytes to copy. - subs r2, r2, #64 - blo 2f - -1: // The main loop copies 64 bytes at a time. - vld1.8 {d0 - d3}, [r1]! - vld1.8 {d4 - d7}, [r1]! - pld [r1, #(CACHE_LINE_SIZE*4)] - subs r2, r2, #64 - vst1.8 {d0 - d3}, [r0, :128]! - vst1.8 {d4 - d7}, [r0, :128]! - bhs 1b - -2: // Fix-up the remaining count and make sure we have >= 32 bytes left. - adds r2, r2, #32 - blo 3f - - // 32 bytes. These cache lines were already preloaded. - vld1.8 {d0 - d3}, [r1]! - sub r2, r2, #32 - vst1.8 {d0 - d3}, [r0, :128]! -3: // Less than 32 left. - add r2, r2, #32 - tst r2, #0x10 - beq copy_less_than_16_unknown_align - // Copies 16 bytes, destination 128 bits aligned. - vld1.8 {d0, d1}, [r1]! - vst1.8 {d0, d1}, [r0, :128]! - -copy_less_than_16_unknown_align: - // Copy up to 15 bytes (count in r2). - movs ip, r2, lsl #29 - bcc 1f - vld1.8 {d0}, [r1]! - vst1.8 {d0}, [r0]! -1: bge 2f - vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! - vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]! - -2: // Copy 0 to 4 bytes. - lsls r2, r2, #31 - itt ne - ldrbne lr, [r1], #1 - strbne lr, [r0], #1 - itttt cs - ldrbcs ip, [r1], #1 - ldrbcs lr, [r1] - strbcs ip, [r0], #1 - strbcs lr, [r0] - - pop {r0, pc} - -check_alignment: - // If src and dst cannot both be aligned to a word boundary, - // use the unaligned copy version. - eor r3, r0, r1 - ands r3, r3, #0x3 - bne copy_unknown_alignment - - // To try and improve performance, stack layout changed, - // i.e., not keeping the stack looking like users expect - // (highest numbered register at highest address). - // TODO: Add debug frame directives. - // We don't need exception unwind directives, because the code below - // does not throw any exceptions and does not call any other functions. - // Generally, newlib functions like this lack debug information for - // assembler source. - .save {r4, r5} - strd r4, r5, [sp, #-8]! - .save {r6, r7} - strd r6, r7, [sp, #-8]! - .save {r8, r9} - strd r8, r9, [sp, #-8]! - - // Optimized for already aligned dst code. - ands ip, r0, #3 - bne dst_not_word_aligned - -word_aligned: - // Align the destination buffer to 8 bytes, to make sure double - // loads and stores don't cross a cache line boundary, - // as they are then more expensive even if the data is in the cache - // (require two load/store issue cycles instead of one). - // If only one of the buffers is not 8 bytes aligned, - // then it's more important to align dst than src, - // because there is more penalty for stores - // than loads that cross a cacheline boundary. - // This check and realignment are only done if there is >= 832 - // bytes to copy. - - // Dst is word aligned, but check if it is already double word aligned. - ands r3, r0, #4 - beq 1f - ldr r3, [r1], #4 - str r3, [r0], #4 - sub r2, #4 - -1: // Can only get here if > 64 bytes to copy, so don't do check r2. - sub r2, #64 - -2: // Every loop iteration copies 64 bytes. - .irp offset, #0, #8, #16, #24, #32 - ldrd r4, r5, [r1, \offset] - strd r4, r5, [r0, \offset] - .endr - - ldrd r4, r5, [r1, #40] - ldrd r6, r7, [r1, #48] - ldrd r8, r9, [r1, #56] - - // Keep the pld as far from the next load as possible. - // The amount to prefetch was determined experimentally using - // large sizes, and verifying the prefetch size does not affect - // the smaller copies too much. - // WARNING: If the ldrd and strd instructions get too far away - // from each other, performance suffers. Three loads - // in a row is the best tradeoff. - pld [r1, #(CACHE_LINE_SIZE*16)] - strd r4, r5, [r0, #40] - strd r6, r7, [r0, #48] - strd r8, r9, [r0, #56] - - add r0, r0, #64 - add r1, r1, #64 - subs r2, r2, #64 - bge 2b - - // Fix-up the remaining count and make sure we have >= 32 bytes left. - adds r2, r2, #32 - blo 4f - - // Copy 32 bytes. These cache lines were already preloaded. - .irp offset, #0, #8, #16, #24 - ldrd r4, r5, [r1, \offset] - strd r4, r5, [r0, \offset] - .endr - add r1, r1, #32 - add r0, r0, #32 - sub r2, r2, #32 -4: // Less than 32 left. - add r2, r2, #32 - tst r2, #0x10 - beq 5f - // Copy 16 bytes. - .irp offset, #0, #8 - ldrd r4, r5, [r1, \offset] - strd r4, r5, [r0, \offset] - .endr - add r1, r1, #16 - add r0, r0, #16 - -5: // Copy up to 15 bytes (count in r2). - movs ip, r2, lsl #29 - bcc 1f - // Copy 8 bytes. - ldrd r4, r5, [r1], #8 - strd r4, r5, [r0], #8 -1: bge 2f - // Copy 4 bytes. - ldr r4, [r1], #4 - str r4, [r0], #4 -2: // Copy 0 to 4 bytes. - lsls r2, r2, #31 - itt ne - ldrbne lr, [r1], #1 - strbne lr, [r0], #1 - itttt cs - ldrbcs ip, [r1], #1 - ldrbcs lr, [r1] - strbcs ip, [r0], #1 - strbcs lr, [r0] - - // Restore registers: optimized pop {r0, pc} - ldrd r8, r9, [sp], #8 - ldrd r6, r7, [sp], #8 - ldrd r4, r5, [sp], #8 - pop {r0, pc} - -dst_not_word_aligned: - // Align dst to word. - rsb ip, ip, #4 - cmp ip, #2 - - itt gt - ldrbgt lr, [r1], #1 - strbgt lr, [r0], #1 - - itt ge - ldrbge lr, [r1], #1 - strbge lr, [r0], #1 - - ldrb lr, [r1], #1 - strb lr, [r0], #1 - - sub r2, r2, ip - - // Src is guaranteed to be at least word aligned by this point. - b word_aligned - - - // Only reached when the __memcpy_chk check fails. -fortify_check_failed: ldr r0, error_message ldr r1, error_code 1: @@ -342,7 +99,7 @@ error_code: .word BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW error_message: .word error_string-(1b+8) -END(memcpy) + .cfi_endproc .data error_string: diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy_base.S b/libc/arch-arm/cortex-a15/bionic/memcpy_base.S new file mode 100644 index 000000000..647e0653f --- /dev/null +++ b/libc/arch-arm/cortex-a15/bionic/memcpy_base.S @@ -0,0 +1,303 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Copyright (c) 2013 ARM Ltd + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the company may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + // Assumes that n >= 0, and dst, src are valid pointers. + // For any sizes less than 832 use the neon code that doesn't + // care about the src alignment. This avoids any checks + // for src alignment, and offers the best improvement since + // smaller sized copies are dominated by the overhead of + // the pre and post main loop. + // For larger copies, if src and dst cannot both be aligned to + // word boundaries, use the neon code. + // For all other copies, align dst to a double word boundary + // and copy using LDRD/STRD instructions. + + cmp r2, #16 + blo .L_copy_less_than_16_unknown_align + + cmp r2, #832 + bge .L_check_alignment + +.L_copy_unknown_alignment: + // Unknown alignment of src and dst. + // Assumes that the first few bytes have already been prefetched. + + // Align destination to 128 bits. The mainloop store instructions + // require this alignment or they will throw an exception. + rsb r3, r0, #0 + ands r3, r3, #0xF + beq 2f + + // Copy up to 15 bytes (count in r3). + sub r2, r2, r3 + movs ip, r3, lsl #31 + + itt mi + ldrbmi lr, [r1], #1 + strbmi lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1], #1 + strbcs ip, [r0], #1 + strbcs lr, [r0], #1 + + movs ip, r3, lsl #29 + bge 1f + // Copies 4 bytes, dst 32 bits aligned before, at least 64 bits after. + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]! +1: bcc 2f + // Copies 8 bytes, dst 64 bits aligned before, at least 128 bits after. + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0, :64]! + +2: // Make sure we have at least 64 bytes to copy. + subs r2, r2, #64 + blo 2f + +1: // The main loop copies 64 bytes at a time. + vld1.8 {d0 - d3}, [r1]! + vld1.8 {d4 - d7}, [r1]! + pld [r1, #(64*4)] + subs r2, r2, #64 + vst1.8 {d0 - d3}, [r0, :128]! + vst1.8 {d4 - d7}, [r0, :128]! + bhs 1b + +2: // Fix-up the remaining count and make sure we have >= 32 bytes left. + adds r2, r2, #32 + blo 3f + + // 32 bytes. These cache lines were already preloaded. + vld1.8 {d0 - d3}, [r1]! + sub r2, r2, #32 + vst1.8 {d0 - d3}, [r0, :128]! +3: // Less than 32 left. + add r2, r2, #32 + tst r2, #0x10 + beq .L_copy_less_than_16_unknown_align + // Copies 16 bytes, destination 128 bits aligned. + vld1.8 {d0, d1}, [r1]! + vst1.8 {d0, d1}, [r0, :128]! + +.L_copy_less_than_16_unknown_align: + // Copy up to 15 bytes (count in r2). + movs ip, r2, lsl #29 + bcc 1f + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0]! +1: bge 2f + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]! + +2: // Copy 0 to 4 bytes. + lsls r2, r2, #31 + itt ne + ldrbne lr, [r1], #1 + strbne lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1] + strbcs ip, [r0], #1 + strbcs lr, [r0] + + pop {r0, pc} + +.L_check_alignment: + // If src and dst cannot both be aligned to a word boundary, + // use the unaligned copy version. + eor r3, r0, r1 + ands r3, r3, #0x3 + bne .L_copy_unknown_alignment + + // To try and improve performance, stack layout changed, + // i.e., not keeping the stack looking like users expect + // (highest numbered register at highest address). + // TODO: Add debug frame directives. + // We don't need exception unwind directives, because the code below + // does not throw any exceptions and does not call any other functions. + // Generally, newlib functions like this lack debug information for + // assembler source. + .save {r4, r5} + strd r4, r5, [sp, #-8]! + .save {r6, r7} + strd r6, r7, [sp, #-8]! + .save {r8, r9} + strd r8, r9, [sp, #-8]! + + // Optimized for already aligned dst code. + ands ip, r0, #3 + bne .L_dst_not_word_aligned + +.L_word_aligned: + // Align the destination buffer to 8 bytes, to make sure double + // loads and stores don't cross a cache line boundary, + // as they are then more expensive even if the data is in the cache + // (require two load/store issue cycles instead of one). + // If only one of the buffers is not 8 bytes aligned, + // then it's more important to align dst than src, + // because there is more penalty for stores + // than loads that cross a cacheline boundary. + // This check and realignment are only done if there is >= 832 + // bytes to copy. + + // Dst is word aligned, but check if it is already double word aligned. + ands r3, r0, #4 + beq 1f + ldr r3, [r1], #4 + str r3, [r0], #4 + sub r2, #4 + +1: // Can only get here if > 64 bytes to copy, so don't do check r2. + sub r2, #64 + +2: // Every loop iteration copies 64 bytes. + .irp offset, #0, #8, #16, #24, #32 + ldrd r4, r5, [r1, \offset] + strd r4, r5, [r0, \offset] + .endr + + ldrd r4, r5, [r1, #40] + ldrd r6, r7, [r1, #48] + ldrd r8, r9, [r1, #56] + + // Keep the pld as far from the next load as possible. + // The amount to prefetch was determined experimentally using + // large sizes, and verifying the prefetch size does not affect + // the smaller copies too much. + // WARNING: If the ldrd and strd instructions get too far away + // from each other, performance suffers. Three loads + // in a row is the best tradeoff. + pld [r1, #(64*16)] + strd r4, r5, [r0, #40] + strd r6, r7, [r0, #48] + strd r8, r9, [r0, #56] + + add r0, r0, #64 + add r1, r1, #64 + subs r2, r2, #64 + bge 2b + + // Fix-up the remaining count and make sure we have >= 32 bytes left. + adds r2, r2, #32 + blo 4f + + // Copy 32 bytes. These cache lines were already preloaded. + .irp offset, #0, #8, #16, #24 + ldrd r4, r5, [r1, \offset] + strd r4, r5, [r0, \offset] + .endr + add r1, r1, #32 + add r0, r0, #32 + sub r2, r2, #32 +4: // Less than 32 left. + add r2, r2, #32 + tst r2, #0x10 + beq 5f + // Copy 16 bytes. + .irp offset, #0, #8 + ldrd r4, r5, [r1, \offset] + strd r4, r5, [r0, \offset] + .endr + add r1, r1, #16 + add r0, r0, #16 + +5: // Copy up to 15 bytes (count in r2). + movs ip, r2, lsl #29 + bcc 1f + // Copy 8 bytes. + ldrd r4, r5, [r1], #8 + strd r4, r5, [r0], #8 +1: bge 2f + // Copy 4 bytes. + ldr r4, [r1], #4 + str r4, [r0], #4 +2: // Copy 0 to 4 bytes. + lsls r2, r2, #31 + itt ne + ldrbne lr, [r1], #1 + strbne lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1] + strbcs ip, [r0], #1 + strbcs lr, [r0] + + // Restore registers: optimized pop {r0, pc} + ldrd r8, r9, [sp], #8 + ldrd r6, r7, [sp], #8 + ldrd r4, r5, [sp], #8 + pop {r0, pc} + +.L_dst_not_word_aligned: + // Align dst to word. + rsb ip, ip, #4 + cmp ip, #2 + + itt gt + ldrbgt lr, [r1], #1 + strbgt lr, [r0], #1 + + itt ge + ldrbge lr, [r1], #1 + strbge lr, [r0], #1 + + ldrb lr, [r1], #1 + strb lr, [r0], #1 + + sub r2, r2, ip + + // Src is guaranteed to be at least word aligned by this point. + b .L_word_aligned diff --git a/libc/arch-arm/cortex-a15/bionic/memset.S b/libc/arch-arm/cortex-a15/bionic/memset.S index 6c143ad81..5593be689 100644 --- a/libc/arch-arm/cortex-a15/bionic/memset.S +++ b/libc/arch-arm/cortex-a15/bionic/memset.S @@ -40,8 +40,14 @@ .syntax unified ENTRY(__memset_chk) + .cfi_startproc cmp r2, r3 - bls done + bls .L_done + + // Preserve lr for backtrace. + push {lr} + .cfi_def_cfa_offset 4 + .cfi_rel_offset lr, 0 ldr r0, error_message ldr r1, error_code @@ -53,24 +59,28 @@ error_code: error_message: .word error_string-(1b+8) + .cfi_endproc END(__memset_chk) ENTRY(bzero) + .cfi_startproc mov r2, r1 mov r1, #0 - -done: +.L_done: // Fall through to memset... + .cfi_endproc END(bzero) ENTRY(memset) - .save {r0} + .cfi_startproc stmfd sp!, {r0} + .cfi_def_cfa_offset 4 + .cfi_rel_offset r0, 0 // The new algorithm is slower for copies < 16 so use the old // neon code in that case. cmp r2, #16 - blo set_less_than_16_unknown_align + blo .L_set_less_than_16_unknown_align // Use strd which requires an even and odd register so move the // values so that: @@ -84,17 +94,17 @@ ENTRY(memset) orr r1, r1, r1, lsr #8 orr r1, r1, r1, lsr #16 -check_alignment: +.L_check_alignment: // Align destination to a double word to avoid the strd crossing // a cache line boundary. ands ip, r3, #7 - bne do_double_word_align + bne .L_do_double_word_align -double_word_aligned: +.L_double_word_aligned: mov r0, r1 subs r2, #64 - blo set_less_than_64 + blo .L_set_less_than_64 1: // Main loop sets 64 bytes at a time. .irp offset, #0, #8, #16, #24, #32, #40, #48, #56 @@ -105,39 +115,39 @@ double_word_aligned: subs r2, #64 bge 1b -set_less_than_64: +.L_set_less_than_64: // Restore r2 to the count of bytes left to set. add r2, #64 lsls ip, r2, #27 - bcc set_less_than_32 + bcc .L_set_less_than_32 // Set 32 bytes. .irp offset, #0, #8, #16, #24 strd r0, r1, [r3, \offset] .endr add r3, #32 -set_less_than_32: - bpl set_less_than_16 +.L_set_less_than_32: + bpl .L_set_less_than_16 // Set 16 bytes. .irp offset, #0, #8 strd r0, r1, [r3, \offset] .endr add r3, #16 -set_less_than_16: +.L_set_less_than_16: // Less than 16 bytes to set. lsls ip, r2, #29 - bcc set_less_than_8 + bcc .L_set_less_than_8 // Set 8 bytes. strd r0, r1, [r3], #8 -set_less_than_8: - bpl set_less_than_4 +.L_set_less_than_8: + bpl .L_set_less_than_4 // Set 4 bytes str r1, [r3], #4 -set_less_than_4: +.L_set_less_than_4: lsls ip, r2, #31 it ne strbne r1, [r3], #1 @@ -148,7 +158,7 @@ set_less_than_4: ldmfd sp!, {r0} bx lr -do_double_word_align: +.L_do_double_word_align: rsb ip, ip, #8 sub r2, r2, ip movs r0, ip, lsl #31 @@ -160,11 +170,11 @@ do_double_word_align: // Dst is at least word aligned by this point. cmp ip, #4 - blo double_word_aligned + blo .L_double_word_aligned str r1, [r3], #4 - b double_word_aligned + b .L_double_word_aligned -set_less_than_16_unknown_align: +.L_set_less_than_16_unknown_align: // Set up to 15 bytes. vdup.8 d0, r1 movs ip, r2, lsl #29 @@ -180,6 +190,7 @@ set_less_than_16_unknown_align: strbcs r1, [r0], #1 ldmfd sp!, {r0} bx lr + .cfi_endproc END(memset) .data diff --git a/libc/arch-arm/cortex-a15/cortex-a15.mk b/libc/arch-arm/cortex-a15/cortex-a15.mk index 281e424ba..c62e7e72a 100644 --- a/libc/arch-arm/cortex-a15/cortex-a15.mk +++ b/libc/arch-arm/cortex-a15/cortex-a15.mk @@ -4,5 +4,7 @@ $(call libc-add-cpu-variant-src,STRCAT,arch-arm/cortex-a15/bionic/strcat.S) $(call libc-add-cpu-variant-src,STRCMP,arch-arm/cortex-a15/bionic/strcmp.S) $(call libc-add-cpu-variant-src,STRCPY,arch-arm/cortex-a15/bionic/strcpy.S) $(call libc-add-cpu-variant-src,STRLEN,arch-arm/cortex-a15/bionic/strlen.S) +$(call libc-add-cpu-variant-src,__STRCAT_CHK,arch-arm/cortex-a15/bionic/__strcat_chk.S) +$(call libc-add-cpu-variant-src,__STRCPY_CHK,arch-arm/cortex-a15/bionic/__strcpy_chk.S) include bionic/libc/arch-arm/generic/generic.mk diff --git a/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S b/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S new file mode 100644 index 000000000..3f866361d --- /dev/null +++ b/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S @@ -0,0 +1,218 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include "libc_events.h" + + .syntax unified + .fpu neon + .thumb + .thumb_func + +// Get the length of src string, then get the source of the dst string. +// Check that the two lengths together don't exceed the threshold, then +// do a memcpy of the data. +ENTRY(__strcat_chk) + .cfi_startproc + pld [r0, #0] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + push {r4, r5} + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 + + mov lr, r2 + + // Save the dst register to r5 + mov r5, r0 + + // Zero out r4 + eor r4, r4, r4 + + // r1 contains the address of the string to count. +.L_strlen_start: + mov r0, r1 + + ands r3, r0, #7 + bne .L_align_src + + .p2align 2 +.L_mainloop: + ldmia r1!, {r2, r3} + + pld [r1, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne .L_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + b .L_mainloop + +.L_zero_in_first_register: + sub r3, r1, r0 + // Check for zero in byte 0. + lsls r2, ip, #17 + beq .L_check_byte1_reg1 + + sub r3, r3, #8 + b .L_finish + +.L_check_byte1_reg1: + bcc .L_check_byte2_reg1 + + sub r3, r3, #7 + b .L_finish + +.L_check_byte2_reg1: + // Check for zero in byte 2. + tst ip, #0x800000 + it ne + subne r3, r3, #6 + bne .L_finish + sub r3, r3, #5 + b .L_finish + +.L_zero_in_second_register: + sub r3, r1, r0 + // Check for zero in byte 0. + lsls r2, ip, #17 + beq .L_check_byte1_reg2 + + sub r3, r3, #4 + b .L_finish + +.L_check_byte1_reg2: + bcc .L_check_byte2_reg2 + + sub r3, r3, #3 + b .L_finish + +.L_check_byte2_reg2: + // Check for zero in byte 2. + tst ip, #0x800000 + it ne + subne r3, r3, #2 + bne .L_finish + sub r3, r3, #1 + b .L_finish + +.L_align_src: + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq .L_align_to_32 + + ldrb r2, [r1], #1 + cbz r2, .L_done + +.L_align_to_32: + bcc .L_align_to_64 + + ldrb r2, [r1], #1 + cbz r2, .L_done + ldrb r2, [r1], #1 + cbz r2, .L_done + +.L_align_to_64: + tst r3, #4 + beq .L_mainloop + ldr r2, [r1], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + b .L_mainloop + +.L_done: + sub r3, r1, r0 + sub r3, r3, #1 + +.L_finish: + cmp r4, #0 + bne .L_strlen_done + + // Time to get the dst string length. + mov r1, r5 + + // Save the original source address to r5. + mov r5, r0 + + // Save the current length (adding 1 for the terminator). + add r4, r3, #1 + b .L_strlen_start + + // r0 holds the pointer to the dst string. + // r3 holds the dst string length. + // r4 holds the src string length + 1. +.L_strlen_done: + add r2, r3, r4 + cmp r2, lr + bgt .L_fortify_check_failed + + // Set up the registers for the memcpy code. + mov r1, r5 + pld [r1, #64] + mov r2, r4 + add r0, r0, r3 + pop {r4, r5} + .cfi_adjust_cfa_offset -8 + .cfi_restore r4 + .cfi_restore r5 + + #include "memcpy_base.S" + +.L_fortify_check_failed: + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 + + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_STRCAT_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+4) + + .cfi_endproc +END(__strcat_chk) + + .data +error_string: + .string "strcat buffer overflow" diff --git a/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S b/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S new file mode 100644 index 000000000..787b0578a --- /dev/null +++ b/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S @@ -0,0 +1,178 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include "libc_events.h" + + .syntax unified + .fpu neon + .thumb + .thumb_func + +// Get the length of the source string first, then do a memcpy of the data +// instead of a strcpy. +ENTRY(__strcpy_chk) + .cfi_startproc + pld [r0, #0] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + + mov lr, r2 + mov r0, r1 + + ands r3, r0, #7 + bne .L_align_src + + .p2align 2 +.L_mainloop: + ldmia r0!, {r2, r3} + + pld [r0, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne .L_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + b .L_mainloop + +.L_zero_in_first_register: + sub r3, r0, r1 + // Check for zero in byte 0. + lsls r2, ip, #17 + beq .L_check_byte1_reg1 + + sub r3, r3, #8 + b .L_check_size + +.L_check_byte1_reg1: + bcc .L_check_byte2_reg1 + + sub r3, r3, #7 + b .L_check_size + +.L_check_byte2_reg1: + // Check for zero in byte 2. + tst ip, #0x800000 + it ne + subne r3, r3, #6 + bne .L_check_size + sub r3, r3, #5 + b .L_check_size + +.L_zero_in_second_register: + sub r3, r0, r1 + // Check for zero in byte 0. + lsls r2, ip, #17 + beq .L_check_byte1_reg2 + + sub r3, r3, #4 + b .L_check_size + +.L_check_byte1_reg2: + bcc .L_check_byte2_reg2 + + sub r3, r3, #3 + b .L_check_size + +.L_check_byte2_reg2: + // Check for zero in byte 2. + tst ip, #0x800000 + it ne + subne r3, r3, #2 + bne .L_check_size + sub r3, r3, #1 + b .L_check_size + +.L_align_src: + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq .L_align_to_32 + + ldrb r2, [r0], #1 + cbz r2, .L_done + +.L_align_to_32: + bcc .L_align_to_64 + + ldrb r2, [r0], #1 + cbz r2, .L_done + ldrb r2, [r0], #1 + cbz r2, .L_done + +.L_align_to_64: + tst r3, #4 + beq .L_mainloop + ldr r2, [r0], #4 + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + b .L_mainloop + +.L_done: + sub r3, r0, r1 + sub r3, r3, #1 + +.L_check_size: + pld [r1, #0] + pld [r1, #64] + ldr r0, [sp] + cmp r3, lr + bge .L_fortify_check_failed + + // Add 1 for copy length to get the string terminator. + add r2, r3, #1 + + #include "memcpy_base.S" + +.L_fortify_check_failed: + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_STRCPY_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+4) + + .cfi_endproc +END(__strcpy_chk) + + .data +error_string: + .string "strcpy buffer overflow" diff --git a/libc/arch-arm/cortex-a9/bionic/memcpy.S b/libc/arch-arm/cortex-a9/bionic/memcpy.S index 4e624d43a..e7beb25b3 100644 --- a/libc/arch-arm/cortex-a9/bionic/memcpy.S +++ b/libc/arch-arm/cortex-a9/bionic/memcpy.S @@ -26,7 +26,6 @@ * SUCH DAMAGE. */ -#include #include #include "libc_events.h" @@ -36,190 +35,40 @@ * cache line. */ - .text + .syntax unified .fpu neon - -#define CACHE_LINE_SIZE 32 + .thumb + .thumb_func ENTRY(__memcpy_chk) + .cfi_startproc cmp r2, r3 - bgt fortify_check_failed + bgt __memcpy_chk_fail // Fall through to memcpy... + .cfi_endproc END(__memcpy_chk) ENTRY(memcpy) - .save {r0, lr} - /* start preloading as early as possible */ - pld [r1, #(CACHE_LINE_SIZE * 0)] - stmfd sp!, {r0, lr} - pld [r1, #(CACHE_LINE_SIZE * 2)] + .cfi_startproc + pld [r1, #0] + stmfd sp!, {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + pld [r1, #64] - // Check so divider is at least 16 bytes, needed for alignment code. - cmp r2, #16 - blo 5f + #include "memcpy_base.S" + .cfi_endproc +END(memcpy) + .cfi_startproc +__memcpy_chk_fail: + // Preserve lr for backtrace. + push {lr} + .cfi_def_cfa_offset 4 + .cfi_rel_offset lr, 0 - /* check if buffers are aligned. If so, run arm-only version */ - eor r3, r0, r1 - ands r3, r3, #0x3 - beq 11f - - /* Check the upper size limit for Neon unaligned memory access in memcpy */ - cmp r2, #224 - blo 3f - - /* align destination to 16 bytes for the write-buffer */ - rsb r3, r0, #0 - ands r3, r3, #0xF - beq 3f - - /* copy up to 15-bytes (count in r3) */ - sub r2, r2, r3 - movs ip, r3, lsl #31 - ldrmib lr, [r1], #1 - strmib lr, [r0], #1 - ldrcsb ip, [r1], #1 - ldrcsb lr, [r1], #1 - strcsb ip, [r0], #1 - strcsb lr, [r0], #1 - movs ip, r3, lsl #29 - bge 1f - // copies 4 bytes, destination 32-bits aligned - vld1.32 {d0[0]}, [r1]! - vst1.32 {d0[0]}, [r0, :32]! -1: bcc 2f - // copies 8 bytes, destination 64-bits aligned - vld1.8 {d0}, [r1]! - vst1.8 {d0}, [r0, :64]! -2: - /* preload immediately the next cache line, which we may need */ - pld [r1, #(CACHE_LINE_SIZE * 0)] - pld [r1, #(CACHE_LINE_SIZE * 2)] -3: - /* make sure we have at least 64 bytes to copy */ - subs r2, r2, #64 - blo 2f - - /* preload all the cache lines we need */ - pld [r1, #(CACHE_LINE_SIZE * 4)] - pld [r1, #(CACHE_LINE_SIZE * 6)] - -1: /* The main loop copies 64 bytes at a time */ - vld1.8 {d0 - d3}, [r1]! - vld1.8 {d4 - d7}, [r1]! - pld [r1, #(CACHE_LINE_SIZE * 6)] - subs r2, r2, #64 - vst1.8 {d0 - d3}, [r0]! - vst1.8 {d4 - d7}, [r0]! - bhs 1b - -2: /* fix-up the remaining count and make sure we have >= 32 bytes left */ - add r2, r2, #64 - subs r2, r2, #32 - blo 4f - -3: /* 32 bytes at a time. These cache lines were already preloaded */ - vld1.8 {d0 - d3}, [r1]! - subs r2, r2, #32 - vst1.8 {d0 - d3}, [r0]! - bhs 3b - -4: /* less than 32 left */ - add r2, r2, #32 - tst r2, #0x10 - beq 5f - // copies 16 bytes, 128-bits aligned - vld1.8 {d0, d1}, [r1]! - vst1.8 {d0, d1}, [r0]! -5: /* copy up to 15-bytes (count in r2) */ - movs ip, r2, lsl #29 - bcc 1f - vld1.8 {d0}, [r1]! - vst1.8 {d0}, [r0]! -1: bge 2f - vld1.32 {d0[0]}, [r1]! - vst1.32 {d0[0]}, [r0]! -2: movs ip, r2, lsl #31 - ldrmib r3, [r1], #1 - ldrcsb ip, [r1], #1 - ldrcsb lr, [r1], #1 - strmib r3, [r0], #1 - strcsb ip, [r0], #1 - strcsb lr, [r0], #1 - - ldmfd sp!, {r0, lr} - bx lr -11: - /* Simple arm-only copy loop to handle aligned copy operations */ - stmfd sp!, {r4, r5, r6, r7, r8} - pld [r1, #(CACHE_LINE_SIZE * 4)] - - /* Check alignment */ - rsb r3, r1, #0 - ands r3, #3 - beq 2f - - /* align source to 32 bits. We need to insert 2 instructions between - * a ldr[b|h] and str[b|h] because byte and half-word instructions - * stall 2 cycles. - */ - movs r12, r3, lsl #31 - sub r2, r2, r3 /* we know that r3 <= r2 because r2 >= 4 */ - ldrmib r3, [r1], #1 - ldrcsb r4, [r1], #1 - ldrcsb r5, [r1], #1 - strmib r3, [r0], #1 - strcsb r4, [r0], #1 - strcsb r5, [r0], #1 - -2: - subs r2, r2, #64 - blt 4f - -3: /* Main copy loop, copying 64 bytes at a time */ - pld [r1, #(CACHE_LINE_SIZE * 8)] - ldmia r1!, {r3, r4, r5, r6, r7, r8, r12, lr} - stmia r0!, {r3, r4, r5, r6, r7, r8, r12, lr} - ldmia r1!, {r3, r4, r5, r6, r7, r8, r12, lr} - stmia r0!, {r3, r4, r5, r6, r7, r8, r12, lr} - subs r2, r2, #64 - bge 3b - -4: /* Check if there are > 32 bytes left */ - adds r2, r2, #64 - subs r2, r2, #32 - blt 5f - - /* Copy 32 bytes */ - ldmia r1!, {r3, r4, r5, r6, r7, r8, r12, lr} - stmia r0!, {r3, r4, r5, r6, r7, r8, r12, lr} - subs r2, #32 - -5: /* Handle any remaining bytes */ - adds r2, #32 - beq 6f - - movs r12, r2, lsl #28 - ldmcsia r1!, {r3, r4, r5, r6} /* 16 bytes */ - ldmmiia r1!, {r7, r8} /* 8 bytes */ - stmcsia r0!, {r3, r4, r5, r6} - stmmiia r0!, {r7, r8} - movs r12, r2, lsl #30 - ldrcs r3, [r1], #4 /* 4 bytes */ - ldrmih r4, [r1], #2 /* 2 bytes */ - strcs r3, [r0], #4 - strmih r4, [r0], #2 - tst r2, #0x1 - ldrneb r3, [r1] /* last byte */ - strneb r3, [r0] -6: - ldmfd sp!, {r4, r5, r6, r7, r8} - ldmfd sp!, {r0, pc} - - - // Only reached when the __memcpy_chk check fails. -fortify_check_failed: ldr r0, error_message ldr r1, error_code 1: @@ -228,8 +77,8 @@ fortify_check_failed: error_code: .word BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW error_message: - .word error_string-(1b+8) -END(memcpy) + .word error_string-(1b+4) + .cfi_endproc .data error_string: diff --git a/libc/arch-arm/cortex-a9/bionic/memcpy_base.S b/libc/arch-arm/cortex-a9/bionic/memcpy_base.S new file mode 100644 index 000000000..46b5a9361 --- /dev/null +++ b/libc/arch-arm/cortex-a9/bionic/memcpy_base.S @@ -0,0 +1,206 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * This code assumes it is running on a processor that supports all arm v7 + * instructions, that supports neon instructions, and that has a 32 byte + * cache line. + */ + + // Check so divider is at least 16 bytes, needed for alignment code. + cmp r2, #16 + blo 5f + + + /* check if buffers are aligned. If so, run arm-only version */ + eor r3, r0, r1 + ands r3, r3, #0x3 + beq 11f + + /* Check the upper size limit for Neon unaligned memory access in memcpy */ + cmp r2, #224 + blo 3f + + /* align destination to 16 bytes for the write-buffer */ + rsb r3, r0, #0 + ands r3, r3, #0xF + beq 3f + + /* copy up to 15-bytes (count in r3) */ + sub r2, r2, r3 + movs ip, r3, lsl #31 + itt mi + ldrbmi lr, [r1], #1 + strbmi lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1], #1 + strbcs ip, [r0], #1 + strbcs lr, [r0], #1 + movs ip, r3, lsl #29 + bge 1f + // copies 4 bytes, destination 32-bits aligned + vld1.32 {d0[0]}, [r1]! + vst1.32 {d0[0]}, [r0, :32]! +1: bcc 2f + // copies 8 bytes, destination 64-bits aligned + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0, :64]! +2: + /* preload immediately the next cache line, which we may need */ + pld [r1, #0] + pld [r1, #(32 * 2)] +3: + /* make sure we have at least 64 bytes to copy */ + subs r2, r2, #64 + blo 2f + + /* preload all the cache lines we need */ + pld [r1, #(32 * 4)] + pld [r1, #(32 * 6)] + +1: /* The main loop copies 64 bytes at a time */ + vld1.8 {d0 - d3}, [r1]! + vld1.8 {d4 - d7}, [r1]! + pld [r1, #(32 * 6)] + subs r2, r2, #64 + vst1.8 {d0 - d3}, [r0]! + vst1.8 {d4 - d7}, [r0]! + bhs 1b + +2: /* fix-up the remaining count and make sure we have >= 32 bytes left */ + add r2, r2, #64 + subs r2, r2, #32 + blo 4f + +3: /* 32 bytes at a time. These cache lines were already preloaded */ + vld1.8 {d0 - d3}, [r1]! + subs r2, r2, #32 + vst1.8 {d0 - d3}, [r0]! + bhs 3b + +4: /* less than 32 left */ + add r2, r2, #32 + tst r2, #0x10 + beq 5f + // copies 16 bytes, 128-bits aligned + vld1.8 {d0, d1}, [r1]! + vst1.8 {d0, d1}, [r0]! +5: /* copy up to 15-bytes (count in r2) */ + movs ip, r2, lsl #29 + bcc 1f + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0]! +1: bge 2f + vld1.32 {d0[0]}, [r1]! + vst1.32 {d0[0]}, [r0]! +2: movs ip, r2, lsl #31 + itt mi + ldrbmi r3, [r1], #1 + strbmi r3, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1], #1 + strbcs ip, [r0], #1 + strbcs lr, [r0], #1 + + ldmfd sp!, {r0, lr} + bx lr +11: + /* Simple arm-only copy loop to handle aligned copy operations */ + stmfd sp!, {r4, r5, r6, r7, r8} + pld [r1, #(32 * 4)] + + /* Check alignment */ + rsb r3, r1, #0 + ands r3, #3 + beq 2f + + /* align source to 32 bits. We need to insert 2 instructions between + * a ldr[b|h] and str[b|h] because byte and half-word instructions + * stall 2 cycles. + */ + movs r12, r3, lsl #31 + sub r2, r2, r3 /* we know that r3 <= r2 because r2 >= 4 */ + itt mi + ldrbmi r3, [r1], #1 + strbmi r3, [r0], #1 + itttt cs + ldrbcs r4, [r1], #1 + ldrbcs r5, [r1], #1 + strbcs r4, [r0], #1 + strbcs r5, [r0], #1 + +2: + subs r2, r2, #64 + blt 4f + +3: /* Main copy loop, copying 64 bytes at a time */ + pld [r1, #(32 * 8)] + ldmia r1!, {r3, r4, r5, r6, r7, r8, r12, lr} + stmia r0!, {r3, r4, r5, r6, r7, r8, r12, lr} + ldmia r1!, {r3, r4, r5, r6, r7, r8, r12, lr} + stmia r0!, {r3, r4, r5, r6, r7, r8, r12, lr} + subs r2, r2, #64 + bge 3b + +4: /* Check if there are > 32 bytes left */ + adds r2, r2, #64 + subs r2, r2, #32 + blt 5f + + /* Copy 32 bytes */ + ldmia r1!, {r3, r4, r5, r6, r7, r8, r12, lr} + stmia r0!, {r3, r4, r5, r6, r7, r8, r12, lr} + subs r2, #32 + +5: /* Handle any remaining bytes */ + adds r2, #32 + beq 6f + + movs r12, r2, lsl #28 + itt cs + ldmiacs r1!, {r3, r4, r5, r6} /* 16 bytes */ + stmiacs r0!, {r3, r4, r5, r6} + itt mi + ldmiami r1!, {r7, r8} /* 8 bytes */ + stmiami r0!, {r7, r8} + movs r12, r2, lsl #30 + itt cs + ldrcs r3, [r1], #4 /* 4 bytes */ + strcs r3, [r0], #4 + itt mi + ldrhmi r4, [r1], #2 /* 2 bytes */ + strhmi r4, [r0], #2 + tst r2, #0x1 + itt ne + ldrbne r3, [r1] /* last byte */ + strbne r3, [r0] +6: + ldmfd sp!, {r4, r5, r6, r7, r8} + ldmfd sp!, {r0, pc} diff --git a/libc/arch-arm/cortex-a9/bionic/memset.S b/libc/arch-arm/cortex-a9/bionic/memset.S index d01143039..bc25a3e2e 100644 --- a/libc/arch-arm/cortex-a9/bionic/memset.S +++ b/libc/arch-arm/cortex-a9/bionic/memset.S @@ -38,8 +38,14 @@ .fpu neon ENTRY(__memset_chk) + .cfi_startproc cmp r2, r3 - bls done + bls .L_done + + // Preserve lr for backtrace. + push {lr} + .cfi_def_cfa_offset 4 + .cfi_rel_offset lr, 0 ldr r0, error_message ldr r1, error_code @@ -51,24 +57,29 @@ error_code: error_message: .word error_string-(1b+8) + .cfi_endproc END(__memset_chk) ENTRY(bzero) + .cfi_startproc mov r2, r1 mov r1, #0 -done: +.L_done: // Fall through to memset... + .cfi_endproc END(bzero) /* memset() returns its first argument. */ ENTRY(memset) + .cfi_startproc # The neon memset only wins for less than 132. cmp r2, #132 bhi 11f - .save {r0} stmfd sp!, {r0} + .cfi_def_cfa_offset 4 + .cfi_rel_offset r0, 0 vdup.8 q0, r1 @@ -106,8 +117,15 @@ ENTRY(memset) * offset = (4-(src&3))&3 = -src & 3 */ - .save {r0, r4-r7, lr} stmfd sp!, {r0, r4-r7, lr} + .cfi_def_cfa_offset 24 + .cfi_rel_offset r0, 0 + .cfi_rel_offset r4, 4 + .cfi_rel_offset r5, 8 + .cfi_rel_offset r6, 12 + .cfi_rel_offset r7, 16 + .cfi_rel_offset lr, 20 + rsb r3, r0, #0 ands r3, r3, #3 cmp r3, r2 @@ -169,6 +187,7 @@ ENTRY(memset) strcsb r1, [r0] ldmfd sp!, {r0, r4-r7, lr} bx lr + .cfi_endproc END(memset) .data diff --git a/libc/arch-arm/cortex-a9/cortex-a9.mk b/libc/arch-arm/cortex-a9/cortex-a9.mk index 61a52c2ac..eee1b3646 100644 --- a/libc/arch-arm/cortex-a9/cortex-a9.mk +++ b/libc/arch-arm/cortex-a9/cortex-a9.mk @@ -4,5 +4,7 @@ $(call libc-add-cpu-variant-src,STRCAT,arch-arm/cortex-a9/bionic/strcat.S) $(call libc-add-cpu-variant-src,STRCMP,arch-arm/cortex-a9/bionic/strcmp.S) $(call libc-add-cpu-variant-src,STRCPY,arch-arm/cortex-a9/bionic/strcpy.S) $(call libc-add-cpu-variant-src,STRLEN,arch-arm/cortex-a9/bionic/strlen.S) +$(call libc-add-cpu-variant-src,__STRCAT_CHK,arch-arm/cortex-a9/bionic/__strcat_chk.S) +$(call libc-add-cpu-variant-src,__STRCPY_CHK,arch-arm/cortex-a9/bionic/__strcpy_chk.S) include bionic/libc/arch-arm/generic/generic.mk diff --git a/libc/arch-arm/generic/generic.mk b/libc/arch-arm/generic/generic.mk index c3a5aa53b..e2300039f 100644 --- a/libc/arch-arm/generic/generic.mk +++ b/libc/arch-arm/generic/generic.mk @@ -4,3 +4,5 @@ $(call libc-add-cpu-variant-src,STRCAT,string/strcat.c) $(call libc-add-cpu-variant-src,STRCMP,arch-arm/generic/bionic/strcmp.S) $(call libc-add-cpu-variant-src,STRCPY,arch-arm/generic/bionic/strcpy.S) $(call libc-add-cpu-variant-src,STRLEN,arch-arm/generic/bionic/strlen.c) +$(call libc-add-cpu-variant-src,__STRCAT_CHK,bionic/__strcat_chk.cpp) +$(call libc-add-cpu-variant-src,__STRCPY_CHK,bionic/__strcpy_chk.cpp) diff --git a/libc/arch-arm/krait/bionic/__strcat_chk.S b/libc/arch-arm/krait/bionic/__strcat_chk.S new file mode 100644 index 000000000..4516d3005 --- /dev/null +++ b/libc/arch-arm/krait/bionic/__strcat_chk.S @@ -0,0 +1,215 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include "libc_events.h" + + .syntax unified + + .thumb + .thumb_func + +// Get the length of src string, then get the source of the dst string. +// Check that the two lengths together don't exceed the threshold, then +// do a memcpy of the data. +ENTRY(__strcat_chk) + .cfi_startproc + pld [r0, #0] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + push {r4, r5} + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 + + mov lr, r2 + + // Save the dst register to r5 + mov r5, r0 + + // Zero out r4 + eor r4, r4, r4 + + // r1 contains the address of the string to count. +.L_strlen_start: + mov r0, r1 + ands r3, r1, #7 + beq .L_mainloop + + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq .L_align_to_32 + + ldrb r2, [r1], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_32: + bcc .L_align_to_64 + ands ip, r3, #2 + beq .L_align_to_64 + + ldrb r2, [r1], #1 + cbz r2, .L_update_count_and_finish + ldrb r2, [r1], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_64: + tst r3, #4 + beq .L_mainloop + ldr r3, [r1], #4 + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + + .p2align 2 +.L_mainloop: + ldrd r2, r3, [r1], #8 + + pld [r1, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne .L_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + b .L_mainloop + +.L_update_count_and_finish: + sub r3, r1, r0 + sub r3, r3, #1 + b .L_finish + +.L_zero_in_first_register: + sub r3, r1, r0 + lsls r2, ip, #17 + bne .L_sub8_and_finish + bcs .L_sub7_and_finish + lsls ip, ip, #1 + bne .L_sub6_and_finish + + sub r3, r3, #5 + b .L_finish + +.L_sub8_and_finish: + sub r3, r3, #8 + b .L_finish + +.L_sub7_and_finish: + sub r3, r3, #7 + b .L_finish + +.L_sub6_and_finish: + sub r3, r3, #6 + b .L_finish + +.L_zero_in_second_register: + sub r3, r1, r0 + lsls r2, ip, #17 + bne .L_sub4_and_finish + bcs .L_sub3_and_finish + lsls ip, ip, #1 + bne .L_sub2_and_finish + + sub r3, r3, #1 + b .L_finish + +.L_sub4_and_finish: + sub r3, r3, #4 + b .L_finish + +.L_sub3_and_finish: + sub r3, r3, #3 + b .L_finish + +.L_sub2_and_finish: + sub r3, r3, #2 + +.L_finish: + cmp r4, #0 + bne .L_strlen_done + + // Time to get the dst string length. + mov r1, r5 + + // Save the original source address to r5. + mov r5, r0 + + // Save the current length (adding 1 for the terminator). + add r4, r3, #1 + b .L_strlen_start + + // r0 holds the pointer to the dst string. + // r3 holds the dst string length. + // r4 holds the src string length + 1. +.L_strlen_done: + add r2, r3, r4 + cmp r2, lr + bgt .L_fortify_check_failed + + // Set up the registers for the memcpy code. + mov r1, r5 + pld [r1, #64] + mov r2, r4 + add r0, r0, r3 + pop {r4, r5} + .cfi_adjust_cfa_offset -8 + .cfi_restore r4 + .cfi_restore r5 + + #include "memcpy_base.S" + +.L_fortify_check_failed: + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 + + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_STRCAT_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+4) + + .cfi_endproc +END(__strcat_chk) + + .data +error_string: + .string "strcat buffer overflow" diff --git a/libc/arch-arm/krait/bionic/__strcpy_chk.S b/libc/arch-arm/krait/bionic/__strcpy_chk.S new file mode 100644 index 000000000..c57268c46 --- /dev/null +++ b/libc/arch-arm/krait/bionic/__strcpy_chk.S @@ -0,0 +1,175 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include "libc_events.h" + + .syntax unified + + .thumb + .thumb_func + +// Get the length of the source string first, then do a memcpy of the data +// instead of a strcpy. +ENTRY(__strcpy_chk) + .cfi_startproc + pld [r0, #0] + push {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + + mov lr, r2 + mov r0, r1 + + ands r3, r1, #7 + beq .L_mainloop + + // Align to a double word (64 bits). + rsb r3, r3, #8 + lsls ip, r3, #31 + beq .L_align_to_32 + + ldrb r2, [r0], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_32: + bcc .L_align_to_64 + ands ip, r3, #2 + beq .L_align_to_64 + + ldrb r2, [r0], #1 + cbz r2, .L_update_count_and_finish + ldrb r2, [r0], #1 + cbz r2, .L_update_count_and_finish + +.L_align_to_64: + tst r3, #4 + beq .L_mainloop + ldr r3, [r0], #4 + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + + .p2align 2 +.L_mainloop: + ldrd r2, r3, [r0], #8 + + pld [r0, #64] + + sub ip, r2, #0x01010101 + bic ip, ip, r2 + ands ip, ip, #0x80808080 + bne .L_zero_in_first_register + + sub ip, r3, #0x01010101 + bic ip, ip, r3 + ands ip, ip, #0x80808080 + bne .L_zero_in_second_register + b .L_mainloop + +.L_update_count_and_finish: + sub r3, r0, r1 + sub r3, r3, #1 + b .L_check_size + +.L_zero_in_first_register: + sub r3, r0, r1 + lsls r2, ip, #17 + bne .L_sub8_and_finish + bcs .L_sub7_and_finish + lsls ip, ip, #1 + bne .L_sub6_and_finish + + sub r3, r3, #5 + b .L_check_size + +.L_sub8_and_finish: + sub r3, r3, #8 + b .L_check_size + +.L_sub7_and_finish: + sub r3, r3, #7 + b .L_check_size + +.L_sub6_and_finish: + sub r3, r3, #6 + b .L_check_size + +.L_zero_in_second_register: + sub r3, r0, r1 + lsls r2, ip, #17 + bne .L_sub4_and_finish + bcs .L_sub3_and_finish + lsls ip, ip, #1 + bne .L_sub2_and_finish + + sub r3, r3, #1 + b .L_check_size + +.L_sub4_and_finish: + sub r3, r3, #4 + b .L_check_size + +.L_sub3_and_finish: + sub r3, r3, #3 + b .L_check_size + +.L_sub2_and_finish: + sub r3, r3, #2 + +.L_check_size: + pld [r1, #0] + pld [r1, #64] + ldr r0, [sp] + cmp r3, lr + bge .L_fortify_check_failed + + // Add 1 for copy length to get the string terminator. + add r2, r3, #1 + + #include "memcpy_base.S" + +.L_fortify_check_failed: + ldr r0, error_message + ldr r1, error_code +1: + add r0, pc + bl __fortify_chk_fail +error_code: + .word BIONIC_EVENT_STRCPY_BUFFER_OVERFLOW +error_message: + .word error_string-(1b+4) + .cfi_endproc +END(__strcpy_chk) + + .data +error_string: + .string "strcpy buffer overflow" diff --git a/libc/arch-arm/krait/bionic/memcpy.S b/libc/arch-arm/krait/bionic/memcpy.S index 3afe18ca8..dde231ae9 100644 --- a/libc/arch-arm/krait/bionic/memcpy.S +++ b/libc/arch-arm/krait/bionic/memcpy.S @@ -28,7 +28,6 @@ /* Assumes neon instructions and a cache line size of 32 bytes. */ -#include #include #include "libc_events.h" @@ -38,103 +37,40 @@ * cache line. */ -#define CACHE_LINE_SIZE 32 - .text + .syntax unified .fpu neon + .thumb + .thumb_func ENTRY(__memcpy_chk) + .cfi_startproc cmp r2, r3 - bgt fortify_check_failed + bgt __memcpy_chk_fail // Fall through to memcpy... + .cfi_endproc END(__memcpy_chk) ENTRY(memcpy) - .save {r0, lr} - /* start preloading as early as possible */ - pld [r1, #(CACHE_LINE_SIZE*4)] - stmfd sp!, {r0, lr} + .cfi_startproc + pld [r1, #64] + stmfd sp!, {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 - /* do we have at least 16-bytes to copy (needed for alignment below) */ - cmp r2, #16 - blo 5f + #include "memcpy_base.S" + .cfi_endproc +END(memcpy) - /* align destination to cache-line for the write-buffer */ - rsb r3, r0, #0 - ands r3, r3, #0xF - beq 2f + .cfi_startproc +__memcpy_chk_fail: + // Preserve lr for backtrace. + push {lr} + .cfi_def_cfa_offset 4 + .cfi_rel_offset lr, 0 - /* copy up to 15-bytes (count in r3) */ - sub r2, r2, r3 - movs ip, r3, lsl #31 - ldrmib lr, [r1], #1 - strmib lr, [r0], #1 - ldrcsb ip, [r1], #1 - ldrcsb lr, [r1], #1 - strcsb ip, [r0], #1 - strcsb lr, [r0], #1 - movs ip, r3, lsl #29 - bge 1f - // copies 4 bytes, destination 32-bits aligned - vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! - vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]! -1: bcc 2f - // copies 8 bytes, destination 64-bits aligned - vld1.8 {d0}, [r1]! - vst1.8 {d0}, [r0, :64]! - -2: /* make sure we have at least 64 bytes to copy */ - subs r2, r2, #64 - blo 2f - -1: /* The main loop copies 64 bytes at a time */ - vld1.8 {d0 - d3}, [r1]! - vld1.8 {d4 - d7}, [r1]! - pld [r1, #(CACHE_LINE_SIZE*2)] - subs r2, r2, #64 - vst1.8 {d0 - d3}, [r0, :128]! - vst1.8 {d4 - d7}, [r0, :128]! - bhs 1b - -2: /* fix-up the remaining count and make sure we have >= 32 bytes left */ - adds r2, r2, #32 - blo 4f - - /* Copy 32 bytes. These cache lines were already preloaded */ - vld1.8 {d0 - d3}, [r1]! - sub r2, r2, #32 - vst1.8 {d0 - d3}, [r0, :128]! - -4: /* less than 32 left */ - add r2, r2, #32 - tst r2, #0x10 - beq 5f - // copies 16 bytes, 128-bits aligned - vld1.8 {d0, d1}, [r1]! - vst1.8 {d0, d1}, [r0, :128]! - -5: /* copy up to 15-bytes (count in r2) */ - movs ip, r2, lsl #29 - bcc 1f - vld1.8 {d0}, [r1]! - vst1.8 {d0}, [r0]! -1: bge 2f - vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! - vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]! -2: movs ip, r2, lsl #31 - ldrmib r3, [r1], #1 - ldrcsb ip, [r1], #1 - ldrcsb lr, [r1], #1 - strmib r3, [r0], #1 - strcsb ip, [r0], #1 - strcsb lr, [r0], #1 - - ldmfd sp!, {r0, lr} - bx lr - - // Only reached when the __memcpy_chk check fails. -fortify_check_failed: ldr r0, error_message ldr r1, error_code 1: @@ -143,8 +79,8 @@ fortify_check_failed: error_code: .word BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW error_message: - .word error_string-(1b+8) -END(memcpy) + .word error_string-(1b+4) + .cfi_endproc .data error_string: diff --git a/libc/arch-arm/krait/bionic/memcpy_base.S b/libc/arch-arm/krait/bionic/memcpy_base.S new file mode 100644 index 000000000..48ce477f0 --- /dev/null +++ b/libc/arch-arm/krait/bionic/memcpy_base.S @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + + +/* + * This code assumes it is running on a processor that supports all arm v7 + * instructions, that supports neon instructions, and that has a 32 byte + * cache line. + */ + +// Assumes neon instructions and a cache line size of 32 bytes. + + /* do we have at least 16-bytes to copy (needed for alignment below) */ + cmp r2, #16 + blo 5f + + /* align destination to cache-line for the write-buffer */ + rsb r3, r0, #0 + ands r3, r3, #0xF + beq 2f + + /* copy up to 15-bytes (count in r3) */ + sub r2, r2, r3 + movs ip, r3, lsl #31 + itt mi + ldrbmi lr, [r1], #1 + strbmi lr, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1], #1 + strbcs ip, [r0], #1 + strbcs lr, [r0], #1 + movs ip, r3, lsl #29 + bge 1f + // copies 4 bytes, destination 32-bits aligned + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]! +1: bcc 2f + // copies 8 bytes, destination 64-bits aligned + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0, :64]! + +2: /* make sure we have at least 64 bytes to copy */ + subs r2, r2, #64 + blo 2f + +1: /* The main loop copies 64 bytes at a time */ + vld1.8 {d0 - d3}, [r1]! + vld1.8 {d4 - d7}, [r1]! + pld [r1, #(32*2)] + subs r2, r2, #64 + vst1.8 {d0 - d3}, [r0, :128]! + vst1.8 {d4 - d7}, [r0, :128]! + bhs 1b + +2: /* fix-up the remaining count and make sure we have >= 32 bytes left */ + adds r2, r2, #32 + blo 4f + + /* Copy 32 bytes. These cache lines were already preloaded */ + vld1.8 {d0 - d3}, [r1]! + sub r2, r2, #32 + vst1.8 {d0 - d3}, [r0, :128]! + +4: /* less than 32 left */ + add r2, r2, #32 + tst r2, #0x10 + beq 5f + // copies 16 bytes, 128-bits aligned + vld1.8 {d0, d1}, [r1]! + vst1.8 {d0, d1}, [r0, :128]! + +5: /* copy up to 15-bytes (count in r2) */ + movs ip, r2, lsl #29 + bcc 1f + vld1.8 {d0}, [r1]! + vst1.8 {d0}, [r0]! +1: bge 2f + vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! + vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]! +2: movs ip, r2, lsl #31 + itt mi + ldrbmi r3, [r1], #1 + strbmi r3, [r0], #1 + itttt cs + ldrbcs ip, [r1], #1 + ldrbcs lr, [r1], #1 + strbcs ip, [r0], #1 + strbcs lr, [r0], #1 + + ldmfd sp!, {r0, lr} + bx lr diff --git a/libc/arch-arm/krait/bionic/memset.S b/libc/arch-arm/krait/bionic/memset.S index 4e4788b7b..15661320b 100644 --- a/libc/arch-arm/krait/bionic/memset.S +++ b/libc/arch-arm/krait/bionic/memset.S @@ -39,8 +39,14 @@ .fpu neon ENTRY(__memset_chk) + .cfi_startproc cmp r2, r3 - bls done + bls .L_done + + // Preserve lr for backtrace. + push {lr} + .cfi_def_cfa_offset 4 + .cfi_rel_offset lr, 0 ldr r0, error_message ldr r1, error_code @@ -52,20 +58,25 @@ error_code: error_message: .word error_string-(1b+8) + .cfi_endproc END(__memset_chk) ENTRY(bzero) + .cfi_startproc mov r2, r1 mov r1, #0 -done: +.L_done: // Fall through to memset... + .cfi_endproc END(bzero) /* memset() returns its first argument. */ ENTRY(memset) - .save {r0} + .cfi_startproc stmfd sp!, {r0} + .cfi_def_cfa_offset 4 + .cfi_rel_offset r0, 0 vdup.8 q0, r1 @@ -98,6 +109,7 @@ ENTRY(memset) strcsb r1, [r0], #1 ldmfd sp!, {r0} bx lr + .cfi_endproc END(memset) .data diff --git a/libc/arch-arm/krait/krait.mk b/libc/arch-arm/krait/krait.mk index 1ff18e9bb..29ab743c6 100644 --- a/libc/arch-arm/krait/krait.mk +++ b/libc/arch-arm/krait/krait.mk @@ -1,6 +1,8 @@ $(call libc-add-cpu-variant-src,MEMCPY,arch-arm/krait/bionic/memcpy.S) $(call libc-add-cpu-variant-src,MEMSET,arch-arm/krait/bionic/memset.S) $(call libc-add-cpu-variant-src,STRCMP,arch-arm/krait/bionic/strcmp.S) +$(call libc-add-cpu-variant-src,__STRCAT_CHK,arch-arm/krait/bionic/__strcat_chk.S) +$(call libc-add-cpu-variant-src,__STRCPY_CHK,arch-arm/krait/bionic/__strcpy_chk.S) # Use cortex-a15 versions of strcat/strcpy/strlen. $(call libc-add-cpu-variant-src,STRCAT,arch-arm/cortex-a15/bionic/strcat.S) $(call libc-add-cpu-variant-src,STRCPY,arch-arm/cortex-a15/bionic/strcpy.S) From 5f7ccea3ffab05aeceecb85c821003cf580630d3 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Tue, 20 Aug 2013 11:20:48 -0700 Subject: [PATCH 57/77] Update all debug directives. The libcorkscrew stack unwinder does not understand cfi directives, so add .save directives so that it can function properly. Also add the directives in to strcmp.S and fix a missing set of directives in cortex-a9/memcpy_base.S. Bug: 10345269 Change-Id: I043f493e0bb6c45bd3f4906fbe1d9f628815b015 --- .../arch-arm/cortex-a15/bionic/__strcat_chk.S | 5 +++- .../arch-arm/cortex-a15/bionic/__strcpy_chk.S | 1 + libc/arch-arm/cortex-a15/bionic/memcpy.S | 5 ++++ libc/arch-arm/cortex-a15/bionic/memset.S | 2 ++ libc/arch-arm/cortex-a15/bionic/strcmp.S | 14 ++++++++++ libc/arch-arm/cortex-a9/bionic/__strcat_chk.S | 3 +++ libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S | 1 + libc/arch-arm/cortex-a9/bionic/memcpy.S | 1 + libc/arch-arm/cortex-a9/bionic/memcpy_base.S | 9 ++++++- libc/arch-arm/cortex-a9/bionic/memset.S | 3 +++ libc/arch-arm/cortex-a9/bionic/strcmp.S | 26 +++++++++++++++++++ libc/arch-arm/krait/bionic/__strcat_chk.S | 3 +++ libc/arch-arm/krait/bionic/__strcpy_chk.S | 1 + libc/arch-arm/krait/bionic/memcpy.S | 4 +++ libc/arch-arm/krait/bionic/memset.S | 2 ++ libc/arch-arm/krait/bionic/strcmp.S | 26 +++++++++++++++++++ 16 files changed, 104 insertions(+), 2 deletions(-) diff --git a/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S b/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S index 08dc78a9a..4b125c89c 100644 --- a/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S +++ b/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S @@ -40,14 +40,16 @@ ENTRY(__strcat_chk) .cfi_startproc pld [r0, #0] + .save {r0, lr} push {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 .cfi_rel_offset lr, 4 + .save {r4, r5} push {r4, r5} .cfi_adjust_cfa_offset 8 .cfi_rel_offset r4, 0 - .cfi_rel_offset r5, 0 + .cfi_rel_offset r5, 4 mov lr, r2 @@ -185,6 +187,7 @@ ENTRY(__strcat_chk) pld [r1, #64] mov r2, r4 add r0, r0, r3 + .pad #-8 pop {r4, r5} .cfi_adjust_cfa_offset -8 .cfi_restore r4 diff --git a/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S b/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S index 9fde59015..a04581681 100644 --- a/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S +++ b/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S @@ -39,6 +39,7 @@ ENTRY(__strcpy_chk) .cfi_startproc pld [r0, #0] + .save {r0, lr} push {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy.S b/libc/arch-arm/cortex-a15/bionic/memcpy.S index 8052d62ef..16881d4de 100644 --- a/libc/arch-arm/cortex-a15/bionic/memcpy.S +++ b/libc/arch-arm/cortex-a15/bionic/memcpy.S @@ -74,18 +74,22 @@ END(__memcpy_chk) ENTRY(memcpy) .cfi_startproc pld [r1, #64] + .save {r0, lr} push {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 .cfi_rel_offset lr, 4 #include "memcpy_base.S" + .cfi_endproc END(memcpy) + .fnstart .cfi_startproc __memcpy_chk_fail: // Preserve lr for backtrace. + .save {lr} push {lr} .cfi_def_cfa_offset 4 .cfi_rel_offset lr, 0 @@ -100,6 +104,7 @@ error_code: error_message: .word error_string-(1b+8) .cfi_endproc + .fnend .data error_string: diff --git a/libc/arch-arm/cortex-a15/bionic/memset.S b/libc/arch-arm/cortex-a15/bionic/memset.S index 5593be689..b5fc6babd 100644 --- a/libc/arch-arm/cortex-a15/bionic/memset.S +++ b/libc/arch-arm/cortex-a15/bionic/memset.S @@ -45,6 +45,7 @@ ENTRY(__memset_chk) bls .L_done // Preserve lr for backtrace. + .save {lr} push {lr} .cfi_def_cfa_offset 4 .cfi_rel_offset lr, 0 @@ -73,6 +74,7 @@ END(bzero) ENTRY(memset) .cfi_startproc + .save {r0} stmfd sp!, {r0} .cfi_def_cfa_offset 4 .cfi_rel_offset r0, 0 diff --git a/libc/arch-arm/cortex-a15/bionic/strcmp.S b/libc/arch-arm/cortex-a15/bionic/strcmp.S index 7aff7c4e1..2719bf7fc 100644 --- a/libc/arch-arm/cortex-a15/bionic/strcmp.S +++ b/libc/arch-arm/cortex-a15/bionic/strcmp.S @@ -122,9 +122,15 @@ ENTRY(strcmp) .macro init /* Macro to save temporary registers and prepare magic values. */ + .save {r4-r7} subs sp, sp, #16 + .cfi_def_cfa_offset 16 strd r4, r5, [sp, #8] + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 strd r6, r7, [sp] + .cfi_rel_offset r6, 8 + .cfi_rel_offset r7, 12 mvn r6, #0 /* all F */ mov r7, #0 /* all 0 */ .endm /* init */ @@ -165,6 +171,7 @@ ENTRY(strcmp) #endif /* not __ARMEB__ */ .endm /* setup_return */ + .cfi_startproc pld [r0, #0] pld [r1, #0] @@ -348,7 +355,13 @@ do_return: /* Restore temporaries early, before computing the return value. */ ldrd r6, r7, [sp] ldrd r4, r5, [sp, #8] + .pad #-16 adds sp, sp, #16 + .cfi_def_cfa_offset 0 + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 /* There is a zero or a different byte between r1 and r2. */ /* r0 contains a mask of all-zero bytes in r1. */ @@ -374,4 +387,5 @@ compute_return_value: it ls sbcls r0, r0, r0 bx lr + .cfi_endproc END(strcmp) diff --git a/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S b/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S index 3f866361d..1329c1612 100644 --- a/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S +++ b/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S @@ -40,10 +40,12 @@ ENTRY(__strcat_chk) .cfi_startproc pld [r0, #0] + .save {r0, lr} push {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 .cfi_rel_offset lr, 4 + .save {r4, r5} push {r4, r5} .cfi_adjust_cfa_offset 8 .cfi_rel_offset r4, 0 @@ -188,6 +190,7 @@ ENTRY(__strcat_chk) pld [r1, #64] mov r2, r4 add r0, r0, r3 + .pad #-8 pop {r4, r5} .cfi_adjust_cfa_offset -8 .cfi_restore r4 diff --git a/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S b/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S index 787b0578a..b697c1f70 100644 --- a/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S +++ b/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S @@ -39,6 +39,7 @@ ENTRY(__strcpy_chk) .cfi_startproc pld [r0, #0] + .save {r0, lr} push {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 diff --git a/libc/arch-arm/cortex-a9/bionic/memcpy.S b/libc/arch-arm/cortex-a9/bionic/memcpy.S index e7beb25b3..ab3b0a006 100644 --- a/libc/arch-arm/cortex-a9/bionic/memcpy.S +++ b/libc/arch-arm/cortex-a9/bionic/memcpy.S @@ -52,6 +52,7 @@ END(__memcpy_chk) ENTRY(memcpy) .cfi_startproc pld [r1, #0] + .save {r0, lr} stmfd sp!, {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 diff --git a/libc/arch-arm/cortex-a9/bionic/memcpy_base.S b/libc/arch-arm/cortex-a9/bionic/memcpy_base.S index 46b5a9361..088d29e35 100644 --- a/libc/arch-arm/cortex-a9/bionic/memcpy_base.S +++ b/libc/arch-arm/cortex-a9/bionic/memcpy_base.S @@ -133,7 +133,14 @@ bx lr 11: /* Simple arm-only copy loop to handle aligned copy operations */ - stmfd sp!, {r4, r5, r6, r7, r8} + .save {r4-r8} + stmfd sp!, {r4-r8} + .cfi_def_cfa_offset 20 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 + .cfi_rel_offset r6, 8 + .cfi_rel_offset r7, 12 + .cfi_rel_offset r8, 16 pld [r1, #(32 * 4)] /* Check alignment */ diff --git a/libc/arch-arm/cortex-a9/bionic/memset.S b/libc/arch-arm/cortex-a9/bionic/memset.S index bc25a3e2e..a7876fb91 100644 --- a/libc/arch-arm/cortex-a9/bionic/memset.S +++ b/libc/arch-arm/cortex-a9/bionic/memset.S @@ -43,6 +43,7 @@ ENTRY(__memset_chk) bls .L_done // Preserve lr for backtrace. + .save {lr} push {lr} .cfi_def_cfa_offset 4 .cfi_rel_offset lr, 0 @@ -77,6 +78,7 @@ ENTRY(memset) cmp r2, #132 bhi 11f + .save {r0} stmfd sp!, {r0} .cfi_def_cfa_offset 4 .cfi_rel_offset r0, 0 @@ -117,6 +119,7 @@ ENTRY(memset) * offset = (4-(src&3))&3 = -src & 3 */ + .save {r0, r4-r7, lr} stmfd sp!, {r0, r4-r7, lr} .cfi_def_cfa_offset 24 .cfi_rel_offset r0, 0 diff --git a/libc/arch-arm/cortex-a9/bionic/strcmp.S b/libc/arch-arm/cortex-a9/bionic/strcmp.S index 9597d0d53..a84c047d5 100644 --- a/libc/arch-arm/cortex-a9/bionic/strcmp.S +++ b/libc/arch-arm/cortex-a9/bionic/strcmp.S @@ -122,9 +122,15 @@ ENTRY(strcmp) .macro init /* Macro to save temporary registers and prepare magic values. */ + .save {r4-r7} subs sp, sp, #16 + .cfi_def_cfa_offset 16 strd r4, r5, [sp, #8] + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 strd r6, r7, [sp] + .cfi_rel_offset r6, 8 + .cfi_rel_offset r7, 12 mvn r6, #0 /* all F */ mov r7, #0 /* all 0 */ .endm /* init */ @@ -165,6 +171,7 @@ ENTRY(strcmp) #endif /* not __ARMEB__ */ .endm /* setup_return */ + .cfi_startproc pld [r0, #0] pld [r1, #0] @@ -334,7 +341,13 @@ do_return: /* Restore temporaries early, before computing the return value. */ ldrd r6, r7, [sp] ldrd r4, r5, [sp, #8] + .pad #-16 adds sp, sp, #16 + .cfi_def_cfa_offset 0 + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 /* There is a zero or a different byte between r1 and r2. */ /* r0 contains a mask of all-zero bytes in r1. */ @@ -519,7 +532,13 @@ strcmp_unaligned: /* Restore registers and stack. */ ldrd r6, r7, [sp] ldrd r4, r5, [sp, #8] + .pad #-16 adds sp, sp, #16 + .cfi_def_cfa_offset 0 + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 bx lr @@ -538,7 +557,14 @@ strcmp_unaligned: /* Restore registers and stack. */ ldrd r6, r7, [sp] ldrd r4, r5, [sp, #8] + .pad #-16 adds sp, sp, #16 + .cfi_def_cfa_offset 0 + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 bx lr + .cfi_endproc END(strcmp) diff --git a/libc/arch-arm/krait/bionic/__strcat_chk.S b/libc/arch-arm/krait/bionic/__strcat_chk.S index 4516d3005..4b125c89c 100644 --- a/libc/arch-arm/krait/bionic/__strcat_chk.S +++ b/libc/arch-arm/krait/bionic/__strcat_chk.S @@ -40,10 +40,12 @@ ENTRY(__strcat_chk) .cfi_startproc pld [r0, #0] + .save {r0, lr} push {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 .cfi_rel_offset lr, 4 + .save {r4, r5} push {r4, r5} .cfi_adjust_cfa_offset 8 .cfi_rel_offset r4, 0 @@ -185,6 +187,7 @@ ENTRY(__strcat_chk) pld [r1, #64] mov r2, r4 add r0, r0, r3 + .pad #-8 pop {r4, r5} .cfi_adjust_cfa_offset -8 .cfi_restore r4 diff --git a/libc/arch-arm/krait/bionic/__strcpy_chk.S b/libc/arch-arm/krait/bionic/__strcpy_chk.S index c57268c46..d5e1db610 100644 --- a/libc/arch-arm/krait/bionic/__strcpy_chk.S +++ b/libc/arch-arm/krait/bionic/__strcpy_chk.S @@ -39,6 +39,7 @@ ENTRY(__strcpy_chk) .cfi_startproc pld [r0, #0] + .save {r0, lr} push {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 diff --git a/libc/arch-arm/krait/bionic/memcpy.S b/libc/arch-arm/krait/bionic/memcpy.S index dde231ae9..093a2443c 100644 --- a/libc/arch-arm/krait/bionic/memcpy.S +++ b/libc/arch-arm/krait/bionic/memcpy.S @@ -55,6 +55,7 @@ END(__memcpy_chk) ENTRY(memcpy) .cfi_startproc pld [r1, #64] + .save {r0, lr} stmfd sp!, {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 @@ -64,9 +65,11 @@ ENTRY(memcpy) .cfi_endproc END(memcpy) + .fnstart .cfi_startproc __memcpy_chk_fail: // Preserve lr for backtrace. + .save {lr} push {lr} .cfi_def_cfa_offset 4 .cfi_rel_offset lr, 0 @@ -81,6 +84,7 @@ error_code: error_message: .word error_string-(1b+4) .cfi_endproc + .fnend .data error_string: diff --git a/libc/arch-arm/krait/bionic/memset.S b/libc/arch-arm/krait/bionic/memset.S index 15661320b..005dfd8cf 100644 --- a/libc/arch-arm/krait/bionic/memset.S +++ b/libc/arch-arm/krait/bionic/memset.S @@ -44,6 +44,7 @@ ENTRY(__memset_chk) bls .L_done // Preserve lr for backtrace. + .save {lr} push {lr} .cfi_def_cfa_offset 4 .cfi_rel_offset lr, 0 @@ -74,6 +75,7 @@ END(bzero) /* memset() returns its first argument. */ ENTRY(memset) .cfi_startproc + .save {r0} stmfd sp!, {r0} .cfi_def_cfa_offset 4 .cfi_rel_offset r0, 0 diff --git a/libc/arch-arm/krait/bionic/strcmp.S b/libc/arch-arm/krait/bionic/strcmp.S index d614b9db0..f26aaf14c 100644 --- a/libc/arch-arm/krait/bionic/strcmp.S +++ b/libc/arch-arm/krait/bionic/strcmp.S @@ -122,9 +122,15 @@ ENTRY(strcmp) .macro init /* Macro to save temporary registers and prepare magic values. */ + .save {r4-r7} subs sp, sp, #16 + .cfi_def_cfa_offset 16 strd r4, r5, [sp, #8] + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 strd r6, r7, [sp] + .cfi_rel_offset r6, 8 + .cfi_rel_offset r7, 12 mvn r6, #0 /* all F */ mov r7, #0 /* all 0 */ .endm /* init */ @@ -165,6 +171,7 @@ ENTRY(strcmp) #endif /* not __ARMEB__ */ .endm /* setup_return */ + .cfi_startproc pld [r0, #0] pld [r1, #0] @@ -347,7 +354,13 @@ do_return: /* Restore temporaries early, before computing the return value. */ ldrd r6, r7, [sp] ldrd r4, r5, [sp, #8] + .pad #-16 adds sp, sp, #16 + .cfi_def_cfa_offset 0 + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 /* There is a zero or a different byte between r1 and r2. */ /* r0 contains a mask of all-zero bytes in r1. */ @@ -452,7 +465,13 @@ miscmp_word_16: /* Restore registers and stack. */ ldrd r6, r7, [sp] ldrd r4, r5, [sp, #8] + .pad #-16 adds sp, sp, #16 + .cfi_def_cfa_offset 0 + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 bx lr @@ -471,7 +490,14 @@ miscmp_word_16: /* Restore registers and stack. */ ldrd r6, r7, [sp] ldrd r4, r5, [sp, #8] + .pad #-16 adds sp, sp, #16 + .cfi_def_cfa_offset 0 + .cfi_restore r4 + .cfi_restore r5 + .cfi_restore r6 + .cfi_restore r7 bx lr + .cfi_endproc END(strcmp) From 9ef1fbb644bf85fa211588853316efa4848547c6 Mon Sep 17 00:00:00 2001 From: Brian Carlstrom Date: Tue, 20 Aug 2013 21:05:44 -0700 Subject: [PATCH 58/77] Make mips_relocate_got tolerate a missing got Bug: 10094803 (cherry picked from commit 7ee26878065abb494600595349ce58b2b2db3709) Change-Id: I9fbb65d20011f2f625fde3b15ac8c6887dd03ae4 --- linker/linker.cpp | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/linker/linker.cpp b/linker/linker.cpp index 386f6dcbd..623be29f3 100644 --- a/linker/linker.cpp +++ b/linker/linker.cpp @@ -1085,17 +1085,15 @@ static int soinfo_relocate(soinfo* si, Elf32_Rel* rel, unsigned count, } #ifdef ANDROID_MIPS_LINKER -static int mips_relocate_got(soinfo* si, soinfo* needed[]) { - unsigned *got; - unsigned local_gotno, gotsym, symtabno; - Elf32_Sym *symtab, *sym; - unsigned g; - - got = si->plt_got; - local_gotno = si->mips_local_gotno; - gotsym = si->mips_gotsym; - symtabno = si->mips_symtabno; - symtab = si->symtab; +static bool mips_relocate_got(soinfo* si, soinfo* needed[]) { + unsigned* got = si->plt_got; + if (got == NULL) { + return true; + } + unsigned local_gotno = si->mips_local_gotno; + unsigned gotsym = si->mips_gotsym; + unsigned symtabno = si->mips_symtabno; + Elf32_Sym* symtab = si->symtab; /* * got[0] is address of lazy resolver function @@ -1106,7 +1104,7 @@ static int mips_relocate_got(soinfo* si, soinfo* needed[]) { */ if ((si->flags & FLAG_LINKER) == 0) { - g = 0; + size_t g = 0; got[g++] = 0xdeadbeef; if (got[g] & 0x80000000) { got[g++] = 0xdeadfeed; @@ -1120,9 +1118,9 @@ static int mips_relocate_got(soinfo* si, soinfo* needed[]) { } /* Now for the global GOT entries */ - sym = symtab + gotsym; + Elf32_Sym* sym = symtab + gotsym; got = si->plt_got + local_gotno; - for (g = gotsym; g < symtabno; g++, sym++, got++) { + for (size_t g = gotsym; g < symtabno; g++, sym++, got++) { const char* sym_name; Elf32_Sym* s; soinfo* lsi; @@ -1136,7 +1134,7 @@ static int mips_relocate_got(soinfo* si, soinfo* needed[]) { s = &symtab[g]; if (ELF32_ST_BIND(s->st_info) != STB_WEAK) { DL_ERR("cannot locate \"%s\"...", sym_name); - return -1; + return false; } *got = 0; } @@ -1148,7 +1146,7 @@ static int mips_relocate_got(soinfo* si, soinfo* needed[]) { *got = lsi->load_bias + s->st_value; } } - return 0; + return true; } #endif @@ -1556,7 +1554,7 @@ static bool soinfo_link_image(soinfo* si) { } #ifdef ANDROID_MIPS_LINKER - if (mips_relocate_got(si, needed)) { + if (!mips_relocate_got(si, needed)) { return false; } #endif From f0c10a4a16c74f5099cf6b272c0a2f11f454e63c Mon Sep 17 00:00:00 2001 From: Elliott Hughes Date: Thu, 22 Aug 2013 11:37:32 -0700 Subject: [PATCH 59/77] Work around tzcode's reliance on signed overflow. I've mailed the tz list about this, and will switch to whatever upstream fix comes along as soon as it's available. Bug: 10310929 (cherry picked from commit 7843d44a594270bcb56e98b130603c054f8a9d38) Change-Id: I205e2440703444c50cecd91d3458d33613ddbc59 --- libc/Android.mk | 2 ++ tests/time_test.cpp | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/libc/Android.mk b/libc/Android.mk index 9610c1400..cc2e0e993 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -718,6 +718,8 @@ LOCAL_CFLAGS := \ -DTZDIR=\"/system/usr/share/zoneinfo\" \ -DTM_GMTOFF=tm_gmtoff \ -DUSG_COMPAT=1 +# tzcode currently relies on signed overflow in numerous places (http://b/10310929). +LOCAL_CFLAGS += -fno-strict-overflow LOCAL_C_INCLUDES := $(libc_common_c_includes) LOCAL_MODULE := libc_tzcode LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk diff --git a/tests/time_test.cpp b/tests/time_test.cpp index 0ad4763a1..02163a554 100644 --- a/tests/time_test.cpp +++ b/tests/time_test.cpp @@ -54,3 +54,16 @@ TEST(time, gmtime) { ASSERT_EQ(0, broken_down->tm_mon); ASSERT_EQ(1970, broken_down->tm_year + 1900); } + +#ifdef __BIONIC__ +TEST(time, mktime_10310929) { + struct tm t; + memset(&t, 0, sizeof(tm)); + t.tm_year = 200; + t.tm_mon = 2; + t.tm_mday = 10; + + ASSERT_EQ(-1, mktime(&t)); + ASSERT_EQ(-1, mktime_tz(&t, "UTC")); +} +#endif From 345eb225ae2e87e36354be5f1e62301179482804 Mon Sep 17 00:00:00 2001 From: Elliott Hughes Date: Thu, 22 Aug 2013 14:13:50 -0700 Subject: [PATCH 60/77] Apply upstream commit 943a6621866e9d6e654f5cfe1494378c1fb8957a. Author: Paul Eggert Date: Thu Aug 22 12:47:51 2013 -0700 * localtime.c: Fix another integer overflow bug in mktime. (time2sub): Avoid undefined behavior on time_t overflow. Reported by Elliott Hughes in . Bug: 10310929 (cherry picked from commit 713fe6463e6ff8cb9689aa8ead88c885d25d03aa) Change-Id: I9ec79fd8d825e6b9e8bb5af549dbfc2182346c05 --- libc/Android.mk | 2 -- libc/tzcode/localtime.c | 8 ++++---- libc/tzcode/private.h | 10 ++++++++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/libc/Android.mk b/libc/Android.mk index cc2e0e993..9610c1400 100644 --- a/libc/Android.mk +++ b/libc/Android.mk @@ -718,8 +718,6 @@ LOCAL_CFLAGS := \ -DTZDIR=\"/system/usr/share/zoneinfo\" \ -DTM_GMTOFF=tm_gmtoff \ -DUSG_COMPAT=1 -# tzcode currently relies on signed overflow in numerous places (http://b/10310929). -LOCAL_CFLAGS += -fno-strict-overflow LOCAL_C_INCLUDES := $(libc_common_c_includes) LOCAL_MODULE := libc_tzcode LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk diff --git a/libc/tzcode/localtime.c b/libc/tzcode/localtime.c index d1b49e5c0..b23eca468 100644 --- a/libc/tzcode/localtime.c +++ b/libc/tzcode/localtime.c @@ -1812,14 +1812,14 @@ time2sub(struct tm * const tmp, } else dir = tmcomp(&mytm, &yourtm); if (dir != 0) { if (t == lo) { - ++t; - if (t <= lo) + if (t == time_t_max) return WRONG; + ++t; ++lo; } else if (t == hi) { - --t; - if (t >= hi) + if (t == time_t_min) return WRONG; + --t; --hi; } if (lo > hi) diff --git a/libc/tzcode/private.h b/libc/tzcode/private.h index a31a26e16..1a938a2ac 100644 --- a/libc/tzcode/private.h +++ b/libc/tzcode/private.h @@ -304,6 +304,16 @@ const char * scheck(const char * string, const char * format); #define TYPE_SIGNED(type) (((type) -1) < 0) #endif /* !defined TYPE_SIGNED */ +/* The minimum and maximum finite time values. */ +static time_t const time_t_min = + (TYPE_SIGNED(time_t) + ? (time_t) -1 << (CHAR_BIT * sizeof (time_t) - 1) + : 0); +static time_t const time_t_max = + (TYPE_SIGNED(time_t) + ? - (~ 0 < 0) - ((time_t) -1 << (CHAR_BIT * sizeof (time_t) - 1)) + : -1); + /* ** Since the definition of TYPE_INTEGRAL contains floating point numbers, ** it cannot be used in preprocessor directives. From d67662b9d6e8854a429b762c83ec37a995c05eb6 Mon Sep 17 00:00:00 2001 From: Robert Greenwalt Date: Thu, 22 Aug 2013 16:56:48 -0700 Subject: [PATCH 61/77] Change how DNS resolver handle no default iface We used to just try any iface we'd been told about as a fallback, but that will end up mistakenly using a secondary network's dns when we really don't have a default connection. It also messed up our detection of whether we were doing the lookup on the default or not (we'd get back our secondary net iface as the default, do the compare and think we were on default). Remove the lies and let dns fail if we don't have an iface for it. bug:10132565 Conflicts: libc/netbsd/resolv/res_cache.c Change-Id: I357a9c34dad83215f44c5e0dd41ce2a7d6fe8f3f --- libc/netbsd/net/getaddrinfo.c | 6 +++--- libc/netbsd/resolv/res_cache.c | 22 ++++++++++------------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/libc/netbsd/net/getaddrinfo.c b/libc/netbsd/net/getaddrinfo.c index 8c1a01b5d..937c42350 100644 --- a/libc/netbsd/net/getaddrinfo.c +++ b/libc/netbsd/net/getaddrinfo.c @@ -1874,10 +1874,10 @@ static bool _using_default_dns(const char *iface) if (iface == NULL || *iface == '\0') return true; if_len = _resolv_get_default_iface(buf, sizeof(buf)); - if (if_len + 1 <= sizeof(buf)) { - if (strcmp(buf, iface) != 0) return false; + if (if_len != 0 && if_len + 1 <= sizeof(buf)) { + if (strcmp(buf, iface) == 0) return true; } - return true; + return false; } /*ARGSUSED*/ diff --git a/libc/netbsd/resolv/res_cache.c b/libc/netbsd/resolv/res_cache.c index 8e1bd14ce..8a6dc83ac 100644 --- a/libc/netbsd/resolv/res_cache.c +++ b/libc/netbsd/resolv/res_cache.c @@ -2531,14 +2531,15 @@ _resolv_set_iface_for_uid_range(const char* ifname, int uid_start, int uid_end) uidiface_info->next = _res_uidiface_list.next; _res_uidiface_list.next = uidiface_info; - XLOG("_resolv_set_iface_for_uid_range: [%d,%d], iface %s\n", low, high, ifname); + XLOG("_resolv_set_iface_for_uid_range: [%d,%d], iface %s\n", uid_start, uid_end, + ifname); } else { XLOG("_resolv_set_iface_for_uid_range failing calloc\n"); rv = -1; errno = EINVAL; } } else { - XLOG("_resolv_set_iface_for_uid_range range [%d,%d] overlaps\n", low, high); + XLOG("_resolv_set_iface_for_uid_range range [%d,%d] overlaps\n", uid_start, uid_end); rv = -1; errno = EINVAL; } @@ -2603,19 +2604,16 @@ _resolv_get_default_iface(char* buff, size_t buffLen) char* ifname = _get_default_iface_locked(); // never null, but may be empty - // if default interface not set. Get first cache with an interface + // if default interface not set give up. if (ifname[0] == '\0') { - ifname = _find_any_iface_name_locked(); // may be null + pthread_mutex_unlock(&_res_cache_list_lock); + return 0; } - size_t len = 0; - // if we got the default iface or if (no-default) the find_any call gave an answer - if (ifname) { - len = strlen(ifname); - if (len < buffLen) { - strncpy(buff, ifname, len); - buff[len] = '\0'; - } + size_t len = strlen(ifname); + if (len < buffLen) { + strncpy(buff, ifname, len); + buff[len] = '\0'; } else { buff[0] = '\0'; } From e9e10c13ff3fdb7104aafd0707b2c851851c9d1f Mon Sep 17 00:00:00 2001 From: Jeff Hao Date: Thu, 15 Aug 2013 14:51:16 -0700 Subject: [PATCH 62/77] Fix pthread_getcpuclockid. clock_gettime was returning EINVAL for the values produced by pthread_getcpuclockid. Bug: 10346183 (cherry picked from commit 9b06cc3c1b2c4e2b08582f3fc9393a05aa589766) Change-Id: Ib81a7024c218a4502f256c3002b9030e2aaa278d --- libc/bionic/pthread_getcpuclockid.cpp | 10 ++++++++-- tests/Android.mk | 2 +- tests/pthread_test.cpp | 10 ++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/libc/bionic/pthread_getcpuclockid.cpp b/libc/bionic/pthread_getcpuclockid.cpp index 10046bab5..d11f56a2a 100644 --- a/libc/bionic/pthread_getcpuclockid.cpp +++ b/libc/bionic/pthread_getcpuclockid.cpp @@ -36,7 +36,13 @@ int pthread_getcpuclockid(pthread_t t, clockid_t* clockid) { return ESRCH; } - enum { CLOCK_IDTYPE_BITS = 3 }; - *clockid = CLOCK_THREAD_CPUTIME_ID | (thread->tid << CLOCK_IDTYPE_BITS); + // The tid is stored in the top bits, but negated. + clockid_t result = ~static_cast(thread->tid) << 3; + // Bits 0 and 1: clock type (0 = CPUCLOCK_PROF, 1 = CPUCLOCK_VIRT, 2 = CPUCLOCK_SCHED). + result |= 2; + // Bit 2: thread (set) or process (clear)? + result |= (1 << 2); + + *clockid = result; return 0; } diff --git a/tests/Android.mk b/tests/Android.mk index 177e4520b..902bf6959 100644 --- a/tests/Android.mk +++ b/tests/Android.mk @@ -157,7 +157,7 @@ include $(CLEAR_VARS) LOCAL_MODULE := bionic-unit-tests-glibc LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk LOCAL_CFLAGS += $(test_c_flags) -LOCAL_LDFLAGS += -lpthread -ldl +LOCAL_LDFLAGS += -lpthread -ldl -lrt LOCAL_LDFLAGS += $(test_dynamic_ldflags) LOCAL_SRC_FILES := $(test_src_files) $(test_dynamic_src_files) LOCAL_STATIC_LIBRARIES += bionic-unit-tests-unwind-test-impl-host diff --git a/tests/pthread_test.cpp b/tests/pthread_test.cpp index c7dbdc78c..d4d38f521 100644 --- a/tests/pthread_test.cpp +++ b/tests/pthread_test.cpp @@ -278,6 +278,16 @@ TEST(pthread, pthread_detach__no_such_thread) { ASSERT_EQ(ESRCH, pthread_detach(dead_thread)); } +TEST(pthread, pthread_getcpuclockid__clock_gettime) { + pthread_t t; + ASSERT_EQ(0, pthread_create(&t, NULL, SleepFn, reinterpret_cast(5))); + + clockid_t c; + ASSERT_EQ(0, pthread_getcpuclockid(t, &c)); + timespec ts; + ASSERT_EQ(0, clock_gettime(c, &ts)); +} + TEST(pthread, pthread_getcpuclockid__no_such_thread) { pthread_t dead_thread; MakeDeadThread(dead_thread); From 05332f2ce7e542d32ff4d5cd9f60248ad71fbf0d Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Wed, 21 Aug 2013 09:41:12 -0700 Subject: [PATCH 63/77] Fix all debug directives. The backtrace when a fortify check failed was not correct. This change adds all of the necessary directives to get a correct backtrace. Fix the strcmp directives and change all labels to local labels. Testing: - Verify that the runtime can decode the stack for __memcpy_chk, __memset_chk, __strcpy_chk, __strcat_chk fortify failures. - Verify that gdb can decode the stack properly when hitting a fortify check. - Verify that the runtime can decode the stack for a seg fault for all of the _chk functions and for memcpy/memset. - Verify that gdb can decode the stack for a seg fault for all of the _chk functions and for memcpy/memset. - Verify that the runtime can decode the stack for a seg fault for strcmp. - Verify that gdb can decode the stack for a seg fault in strcmp. Bug: 10342460 Bug: 10345269 Change-Id: I1dedadfee207dce4a285e17a21e8952bbc63786a --- .../arch-arm/cortex-a15/bionic/__strcat_chk.S | 29 ++++-- .../arch-arm/cortex-a15/bionic/__strcpy_chk.S | 21 +++- libc/arch-arm/cortex-a15/bionic/memcpy.S | 15 +-- libc/arch-arm/cortex-a15/bionic/memcpy_base.S | 40 ++++++-- libc/arch-arm/cortex-a15/bionic/strcmp.S | 95 +++++++++--------- libc/arch-arm/cortex-a9/bionic/__strcat_chk.S | 29 ++++-- libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S | 25 ++++- libc/arch-arm/cortex-a9/bionic/memcpy.S | 12 ++- libc/arch-arm/cortex-a9/bionic/memcpy_base.S | 32 ++++-- libc/arch-arm/cortex-a9/bionic/memset.S | 20 ++-- libc/arch-arm/cortex-a9/bionic/strcmp.S | 94 ++++++++---------- libc/arch-arm/krait/bionic/__strcat_chk.S | 27 ++++-- libc/arch-arm/krait/bionic/__strcpy_chk.S | 21 +++- libc/arch-arm/krait/bionic/memcpy.S | 15 +-- libc/arch-arm/krait/bionic/memcpy_base.S | 10 ++ libc/arch-arm/krait/bionic/strcmp.S | 97 +++++++++---------- 16 files changed, 350 insertions(+), 232 deletions(-) diff --git a/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S b/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S index 4b125c89c..46936005d 100644 --- a/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S +++ b/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S @@ -40,13 +40,13 @@ ENTRY(__strcat_chk) .cfi_startproc pld [r0, #0] - .save {r0, lr} push {r0, lr} + .save {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 .cfi_rel_offset lr, 4 - .save {r4, r5} push {r4, r5} + .save {r4, r5} .cfi_adjust_cfa_offset 8 .cfi_rel_offset r4, 0 .cfi_rel_offset r5, 4 @@ -180,22 +180,31 @@ ENTRY(__strcat_chk) .L_strlen_done: add r2, r3, r4 cmp r2, lr - bgt .L_fortify_check_failed + bgt __strcat_chk_failed // Set up the registers for the memcpy code. mov r1, r5 pld [r1, #64] mov r2, r4 add r0, r0, r3 - .pad #-8 pop {r4, r5} - .cfi_adjust_cfa_offset -8 - .cfi_restore r4 - .cfi_restore r5 - #include "memcpy_base.S" + .cfi_endproc +END(__strcat_chk) -.L_fortify_check_failed: +#define MEMCPY_BASE __strcat_chk_memcpy_base +#define MEMCPY_BASE_ALIGNED __strcat_chk_memcpy_base_aligned + +#include "memcpy_base.S" + +ENTRY(__strcat_chk_failed) + .cfi_startproc + .save {r0, lr} + .save {r4, r5} + + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 .cfi_adjust_cfa_offset 8 .cfi_rel_offset r4, 0 .cfi_rel_offset r5, 4 @@ -211,7 +220,7 @@ error_message: .word error_string-(1b+4) .cfi_endproc -END(__strcat_chk) +END(__strcat_chk_failed) .data error_string: diff --git a/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S b/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S index a04581681..1224b491d 100644 --- a/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S +++ b/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S @@ -39,8 +39,8 @@ ENTRY(__strcpy_chk) .cfi_startproc pld [r0, #0] - .save {r0, lr} push {r0, lr} + .save {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 .cfi_rel_offset lr, 4 @@ -151,14 +151,25 @@ ENTRY(__strcpy_chk) pld [r1, #64] ldr r0, [sp] cmp r3, lr - bge .L_fortify_check_failed + bge __strcpy_chk_failed // Add 1 for copy length to get the string terminator. add r2, r3, #1 - #include "memcpy_base.S" + .cfi_endproc +END(__strcpy_chk) + +#define MEMCPY_BASE __strcpy_chk_memcpy_base +#define MEMCPY_BASE_ALIGNED __strcpy_chk_memcpy_base_aligned +#include "memcpy_base.S" + +ENTRY(__strcpy_chk_failed) + .cfi_startproc + .save {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 -.L_fortify_check_failed: ldr r0, error_message ldr r1, error_code 1: @@ -170,7 +181,7 @@ error_message: .word error_string-(1b+4) .cfi_endproc -END(__strcpy_chk) +END(__strcpy_chk_failed) .data error_string: diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy.S b/libc/arch-arm/cortex-a15/bionic/memcpy.S index 16881d4de..a300e43db 100644 --- a/libc/arch-arm/cortex-a15/bionic/memcpy.S +++ b/libc/arch-arm/cortex-a15/bionic/memcpy.S @@ -74,23 +74,24 @@ END(__memcpy_chk) ENTRY(memcpy) .cfi_startproc pld [r1, #64] - .save {r0, lr} push {r0, lr} + .save {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 .cfi_rel_offset lr, 4 - #include "memcpy_base.S" - .cfi_endproc END(memcpy) - .fnstart +#define MEMCPY_BASE __memcpy_base +#define MEMCPY_BASE_ALIGNED __memcpy_base_aligned +#include "memcpy_base.S" + +ENTRY(__memcpy_chk_fail) .cfi_startproc -__memcpy_chk_fail: // Preserve lr for backtrace. - .save {lr} push {lr} + .save {lr} .cfi_def_cfa_offset 4 .cfi_rel_offset lr, 0 @@ -104,7 +105,7 @@ error_code: error_message: .word error_string-(1b+8) .cfi_endproc - .fnend +END(__memcpy_chk_fail) .data error_string: diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy_base.S b/libc/arch-arm/cortex-a15/bionic/memcpy_base.S index 647e0653f..015467607 100644 --- a/libc/arch-arm/cortex-a15/bionic/memcpy_base.S +++ b/libc/arch-arm/cortex-a15/bionic/memcpy_base.S @@ -53,6 +53,13 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +ENTRY(MEMCPY_BASE) + .cfi_startproc + .save {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + // Assumes that n >= 0, and dst, src are valid pointers. // For any sizes less than 832 use the neon code that doesn't // care about the src alignment. This avoids any checks @@ -162,20 +169,34 @@ ands r3, r3, #0x3 bne .L_copy_unknown_alignment + .cfi_endproc +END(MEMCPY_BASE) + +ENTRY(MEMCPY_BASE_ALIGNED) + .cfi_startproc + .save {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + // To try and improve performance, stack layout changed, // i.e., not keeping the stack looking like users expect // (highest numbered register at highest address). - // TODO: Add debug frame directives. - // We don't need exception unwind directives, because the code below - // does not throw any exceptions and does not call any other functions. - // Generally, newlib functions like this lack debug information for - // assembler source. - .save {r4, r5} strd r4, r5, [sp, #-8]! - .save {r6, r7} + .save {r4, r5} + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r4, 0 + .cfi_rel_offset r5, 4 strd r6, r7, [sp, #-8]! - .save {r8, r9} + .save {r6, r7} + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r6, 0 + .cfi_rel_offset r7, 0 strd r8, r9, [sp, #-8]! + .save {r8, r9} + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset r8, 0 + .cfi_rel_offset r9, 4 // Optimized for already aligned dst code. ands ip, r0, #3 @@ -301,3 +322,6 @@ // Src is guaranteed to be at least word aligned by this point. b .L_word_aligned + + .cfi_endproc +END(MEMCPY_BASE_ALIGNED) diff --git a/libc/arch-arm/cortex-a15/bionic/strcmp.S b/libc/arch-arm/cortex-a15/bionic/strcmp.S index 2719bf7fc..13b329f4f 100644 --- a/libc/arch-arm/cortex-a15/bionic/strcmp.S +++ b/libc/arch-arm/cortex-a15/bionic/strcmp.S @@ -122,7 +122,6 @@ ENTRY(strcmp) .macro init /* Macro to save temporary registers and prepare magic values. */ - .save {r4-r7} subs sp, sp, #16 .cfi_def_cfa_offset 16 strd r4, r5, [sp, #8] @@ -178,12 +177,13 @@ ENTRY(strcmp) /* Are both strings double-word aligned? */ orr ip, r0, r1 tst ip, #7 - bne do_align + bne .L_do_align /* Fast path. */ + .save {r4-r7} init -doubleword_aligned: +.L_doubleword_aligned: /* Get here when the strings to compare are double-word aligned. */ /* Compare two words in every iteration. */ @@ -196,14 +196,14 @@ doubleword_aligned: ldrd r2, r3, [r0], #8 ldrd r4, r5, [r1], #8 - magic_compare_and_branch w1=r2, w2=r4, label=return_24 - magic_compare_and_branch w1=r3, w2=r5, label=return_35 + magic_compare_and_branch w1=r2, w2=r4, label=.L_return_24 + magic_compare_and_branch w1=r3, w2=r5, label=.L_return_35 b 2b -do_align: +.L_do_align: /* Is the first string word-aligned? */ ands ip, r0, #3 - beq word_aligned_r0 + beq .L_word_aligned_r0 /* Fast compare byte by byte until the first string is word-aligned. */ /* The offset of r0 from a word boundary is in ip. Thus, the number of bytes @@ -211,58 +211,58 @@ do_align: bic r0, r0, #3 ldr r2, [r0], #4 lsls ip, ip, #31 - beq byte2 - bcs byte3 + beq .L_byte2 + bcs .L_byte3 -byte1: +.L_byte1: ldrb ip, [r1], #1 uxtb r3, r2, ror #BYTE1_OFFSET subs ip, r3, ip - bne fast_return - m_cbz reg=r3, label=fast_return + bne .L_fast_return + m_cbz reg=r3, label=.L_fast_return -byte2: +.L_byte2: ldrb ip, [r1], #1 uxtb r3, r2, ror #BYTE2_OFFSET subs ip, r3, ip - bne fast_return - m_cbz reg=r3, label=fast_return + bne .L_fast_return + m_cbz reg=r3, label=.L_fast_return -byte3: +.L_byte3: ldrb ip, [r1], #1 uxtb r3, r2, ror #BYTE3_OFFSET subs ip, r3, ip - bne fast_return - m_cbnz reg=r3, label=word_aligned_r0 + bne .L_fast_return + m_cbnz reg=r3, label=.L_word_aligned_r0 -fast_return: +.L_fast_return: mov r0, ip bx lr -word_aligned_r0: +.L_word_aligned_r0: init /* The first string is word-aligned. */ /* Is the second string word-aligned? */ ands ip, r1, #3 - bne strcmp_unaligned + bne .L_strcmp_unaligned -word_aligned: +.L_word_aligned: /* The strings are word-aligned. */ /* Is the first string double-word aligned? */ tst r0, #4 - beq doubleword_aligned_r0 + beq .L_doubleword_aligned_r0 /* If r0 is not double-word aligned yet, align it by loading and comparing the next word from each string. */ ldr r2, [r0], #4 ldr r4, [r1], #4 - magic_compare_and_branch w1=r2 w2=r4 label=return_24 + magic_compare_and_branch w1=r2 w2=r4 label=.L_return_24 -doubleword_aligned_r0: +.L_doubleword_aligned_r0: /* Get here when r0 is double-word aligned. */ /* Is r1 doubleword_aligned? */ tst r1, #4 - beq doubleword_aligned + beq .L_doubleword_aligned /* Get here when the strings to compare are word-aligned, r0 is double-word aligned, but r1 is not double-word aligned. */ @@ -278,9 +278,9 @@ doubleword_aligned_r0: /* Load the next double-word from each string and compare. */ ldrd r2, r3, [r0], #8 - magic_compare_and_branch w1=r2 w2=r5 label=return_25 + magic_compare_and_branch w1=r2 w2=r5 label=.L_return_25 ldrd r4, r5, [r1], #8 - magic_compare_and_branch w1=r3 w2=r4 label=return_34 + magic_compare_and_branch w1=r3 w2=r4 label=.L_return_34 b 3b .macro miscmp_word offsetlo offsethi @@ -304,47 +304,47 @@ doubleword_aligned_r0: and r2, r3, r6, S2LOMEM #\offsetlo it eq cmpeq r2, r5 - bne return_25 + bne .L_return_25 ldr r5, [r1], #4 cmp ip, #0 eor r3, r2, r3 S2HIMEM r2, r5, #\offsethi it eq cmpeq r3, r2 - bne return_32 + bne .L_return_32 b 7b .endm /* miscmp_word */ -strcmp_unaligned: +.L_strcmp_unaligned: /* r0 is word-aligned, r1 is at offset ip from a word. */ /* Align r1 to the (previous) word-boundary. */ bic r1, r1, #3 /* Unaligned comparison word by word using LDRs. */ cmp ip, #2 - beq miscmp_word_16 /* If ip == 2. */ - bge miscmp_word_24 /* If ip == 3. */ + beq .L_miscmp_word_16 /* If ip == 2. */ + bge .L_miscmp_word_24 /* If ip == 3. */ miscmp_word offsetlo=8 offsethi=24 /* If ip == 1. */ -miscmp_word_16: miscmp_word offsetlo=16 offsethi=16 -miscmp_word_24: miscmp_word offsetlo=24 offsethi=8 +.L_miscmp_word_16: miscmp_word offsetlo=16 offsethi=16 +.L_miscmp_word_24: miscmp_word offsetlo=24 offsethi=8 -return_32: +.L_return_32: setup_return w1=r3, w2=r2 - b do_return -return_34: + b .L_do_return +.L_return_34: setup_return w1=r3, w2=r4 - b do_return -return_25: + b .L_do_return +.L_return_25: setup_return w1=r2, w2=r5 - b do_return -return_35: + b .L_do_return +.L_return_35: setup_return w1=r3, w2=r5 - b do_return -return_24: + b .L_do_return +.L_return_24: setup_return w1=r2, w2=r4 -do_return: +.L_do_return: #ifdef __ARMEB__ mov r0, ip @@ -355,7 +355,6 @@ do_return: /* Restore temporaries early, before computing the return value. */ ldrd r6, r7, [sp] ldrd r4, r5, [sp, #8] - .pad #-16 adds sp, sp, #16 .cfi_def_cfa_offset 0 .cfi_restore r4 @@ -366,7 +365,7 @@ do_return: /* There is a zero or a different byte between r1 and r2. */ /* r0 contains a mask of all-zero bytes in r1. */ /* Using r0 and not ip here because cbz requires low register. */ - m_cbz reg=r0, label=compute_return_value + m_cbz reg=r0, label=.L_compute_return_value clz r0, r0 /* r0 contains the number of bits on the left of the first all-zero byte in r1. */ rsb r0, r0, #24 @@ -374,7 +373,7 @@ do_return: lsr r1, r1, r0 lsr r2, r2, r0 -compute_return_value: +.L_compute_return_value: movs r0, #1 cmp r1, r2 /* The return value is computed as follows. diff --git a/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S b/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S index 1329c1612..cc43456c3 100644 --- a/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S +++ b/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S @@ -40,13 +40,13 @@ ENTRY(__strcat_chk) .cfi_startproc pld [r0, #0] - .save {r0, lr} push {r0, lr} + .save {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 .cfi_rel_offset lr, 4 - .save {r4, r5} push {r4, r5} + .save {r4, r5} .cfi_adjust_cfa_offset 8 .cfi_rel_offset r4, 0 .cfi_rel_offset r5, 4 @@ -183,22 +183,31 @@ ENTRY(__strcat_chk) .L_strlen_done: add r2, r3, r4 cmp r2, lr - bgt .L_fortify_check_failed + bgt __strcat_chk_fail // Set up the registers for the memcpy code. mov r1, r5 pld [r1, #64] mov r2, r4 add r0, r0, r3 - .pad #-8 pop {r4, r5} - .cfi_adjust_cfa_offset -8 - .cfi_restore r4 - .cfi_restore r5 - #include "memcpy_base.S" + // Fall through into the memcpy_base function. + .cfi_endproc +END(__strcat_chk) -.L_fortify_check_failed: +#define MEMCPY_BASE __strcat_chk_memcpy_base +#define MEMCPY_BASE_ALIGNED __strcat_chk_memcpy_base_aligned +#include "memcpy_base.S" + +ENTRY(__strcat_chk_fail) + .cfi_startproc + + .save {r0, lr} + .save {r4, r5} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 .cfi_adjust_cfa_offset 8 .cfi_rel_offset r4, 0 .cfi_rel_offset r5, 4 @@ -214,7 +223,7 @@ error_message: .word error_string-(1b+4) .cfi_endproc -END(__strcat_chk) +END(__strcat_chk_fail) .data error_string: diff --git a/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S b/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S index b697c1f70..dd3370bae 100644 --- a/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S +++ b/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S @@ -39,8 +39,8 @@ ENTRY(__strcpy_chk) .cfi_startproc pld [r0, #0] - .save {r0, lr} push {r0, lr} + .save {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 .cfi_rel_offset lr, 4 @@ -153,26 +153,41 @@ ENTRY(__strcpy_chk) pld [r1, #64] ldr r0, [sp] cmp r3, lr - bge .L_fortify_check_failed + bge __strcpy_chk_fail // Add 1 for copy length to get the string terminator. add r2, r3, #1 - #include "memcpy_base.S" + .cfi_endproc + + // Fall through into the memcpy_base function. +END(__strcpy_chk) + +#define MEMCPY_BASE __strcpy_chk_memcpy_base +#define MEMCPY_BASE_ALIGNED __strcpy_chk_memcpy_base_aligned +#include "memcpy_base.S" + +ENTRY(__strcpy_chk_fail) + .cfi_startproc + + .save {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 -.L_fortify_check_failed: ldr r0, error_message ldr r1, error_code 1: add r0, pc bl __fortify_chk_fail + error_code: .word BIONIC_EVENT_STRCPY_BUFFER_OVERFLOW error_message: .word error_string-(1b+4) .cfi_endproc -END(__strcpy_chk) +END(__strcpy_chk_fail) .data error_string: diff --git a/libc/arch-arm/cortex-a9/bionic/memcpy.S b/libc/arch-arm/cortex-a9/bionic/memcpy.S index ab3b0a006..21e0ebea6 100644 --- a/libc/arch-arm/cortex-a9/bionic/memcpy.S +++ b/libc/arch-arm/cortex-a9/bionic/memcpy.S @@ -51,22 +51,27 @@ END(__memcpy_chk) ENTRY(memcpy) .cfi_startproc + pld [r1, #0] - .save {r0, lr} stmfd sp!, {r0, lr} + .save {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 .cfi_rel_offset lr, 4 pld [r1, #64] - #include "memcpy_base.S" .cfi_endproc END(memcpy) +#define MEMCPY_BASE __memcpy_base +#define MEMCPY_BASE_ALIGNED __memcpy_base_aligned +#include "memcpy_base.S" + +ENTRY(__memcpy_chk_fail) .cfi_startproc -__memcpy_chk_fail: // Preserve lr for backtrace. push {lr} + .save {lr} .cfi_def_cfa_offset 4 .cfi_rel_offset lr, 0 @@ -80,6 +85,7 @@ error_code: error_message: .word error_string-(1b+4) .cfi_endproc +END(__memcpy_chk_fail) .data error_string: diff --git a/libc/arch-arm/cortex-a9/bionic/memcpy_base.S b/libc/arch-arm/cortex-a9/bionic/memcpy_base.S index 088d29e35..e8ff4f5b2 100644 --- a/libc/arch-arm/cortex-a9/bionic/memcpy_base.S +++ b/libc/arch-arm/cortex-a9/bionic/memcpy_base.S @@ -32,15 +32,21 @@ * cache line. */ +ENTRY(MEMCPY_BASE) + .cfi_startproc + .save {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + // Check so divider is at least 16 bytes, needed for alignment code. cmp r2, #16 blo 5f - /* check if buffers are aligned. If so, run arm-only version */ eor r3, r0, r1 ands r3, r3, #0x3 - beq 11f + beq __memcpy_base_aligned /* Check the upper size limit for Neon unaligned memory access in memcpy */ cmp r2, #224 @@ -131,11 +137,22 @@ ldmfd sp!, {r0, lr} bx lr -11: + + .cfi_endproc +END(MEMCPY_BASE) + +ENTRY(MEMCPY_BASE_ALIGNED) + .cfi_startproc + + .save {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + /* Simple arm-only copy loop to handle aligned copy operations */ - .save {r4-r8} stmfd sp!, {r4-r8} - .cfi_def_cfa_offset 20 + .save {r4-r8} + .cfi_adjust_cfa_offset 20 .cfi_rel_offset r4, 0 .cfi_rel_offset r5, 4 .cfi_rel_offset r6, 8 @@ -209,5 +226,8 @@ ldrbne r3, [r1] /* last byte */ strbne r3, [r0] 6: - ldmfd sp!, {r4, r5, r6, r7, r8} + ldmfd sp!, {r4-r8} ldmfd sp!, {r0, pc} + + .cfi_endproc +END(MEMCPY_BASE_ALIGNED) diff --git a/libc/arch-arm/cortex-a9/bionic/memset.S b/libc/arch-arm/cortex-a9/bionic/memset.S index a7876fb91..87d2c088e 100644 --- a/libc/arch-arm/cortex-a9/bionic/memset.S +++ b/libc/arch-arm/cortex-a9/bionic/memset.S @@ -43,8 +43,8 @@ ENTRY(__memset_chk) bls .L_done // Preserve lr for backtrace. - .save {lr} push {lr} + .save {lr} .cfi_def_cfa_offset 4 .cfi_rel_offset lr, 0 @@ -74,12 +74,13 @@ END(bzero) /* memset() returns its first argument. */ ENTRY(memset) .cfi_startproc + # The neon memset only wins for less than 132. cmp r2, #132 - bhi 11f + bhi __memset_large_copy - .save {r0} stmfd sp!, {r0} + .save {r0} .cfi_def_cfa_offset 4 .cfi_rel_offset r0, 0 @@ -114,13 +115,18 @@ ENTRY(memset) strcsb r1, [r0], #1 ldmfd sp!, {r0} bx lr -11: + + .cfi_endproc +END(memset) + +ENTRY(__memset_large_copy) + .cfi_startproc + /* compute the offset to align the destination * offset = (4-(src&3))&3 = -src & 3 */ - - .save {r0, r4-r7, lr} stmfd sp!, {r0, r4-r7, lr} + .save {r0, r4-r7, lr} .cfi_def_cfa_offset 24 .cfi_rel_offset r0, 0 .cfi_rel_offset r4, 4 @@ -191,7 +197,7 @@ ENTRY(memset) ldmfd sp!, {r0, r4-r7, lr} bx lr .cfi_endproc -END(memset) +END(__memset_large_copy) .data error_string: diff --git a/libc/arch-arm/cortex-a9/bionic/strcmp.S b/libc/arch-arm/cortex-a9/bionic/strcmp.S index a84c047d5..232df75b6 100644 --- a/libc/arch-arm/cortex-a9/bionic/strcmp.S +++ b/libc/arch-arm/cortex-a9/bionic/strcmp.S @@ -122,7 +122,6 @@ ENTRY(strcmp) .macro init /* Macro to save temporary registers and prepare magic values. */ - .save {r4-r7} subs sp, sp, #16 .cfi_def_cfa_offset 16 strd r4, r5, [sp, #8] @@ -178,12 +177,13 @@ ENTRY(strcmp) /* Are both strings double-word aligned? */ orr ip, r0, r1 tst ip, #7 - bne do_align + bne .L_do_align /* Fast path. */ + .save {r4-r7} init -doubleword_aligned: +.L_doubleword_aligned: /* Get here when the strings to compare are double-word aligned. */ /* Compare two words in every iteration. */ @@ -196,14 +196,14 @@ doubleword_aligned: ldrd r2, r3, [r0], #8 ldrd r4, r5, [r1], #8 - magic_compare_and_branch w1=r2, w2=r4, label=return_24 - magic_compare_and_branch w1=r3, w2=r5, label=return_35 + magic_compare_and_branch w1=r2, w2=r4, label=.L_return_24 + magic_compare_and_branch w1=r3, w2=r5, label=.L_return_35 b 2b -do_align: +.L_do_align: /* Is the first string word-aligned? */ ands ip, r0, #3 - beq word_aligned_r0 + beq .L_word_aligned_r0 /* Fast compare byte by byte until the first string is word-aligned. */ /* The offset of r0 from a word boundary is in ip. Thus, the number of bytes @@ -211,58 +211,58 @@ do_align: bic r0, r0, #3 ldr r2, [r0], #4 lsls ip, ip, #31 - beq byte2 - bcs byte3 + beq .L_byte2 + bcs .L_byte3 -byte1: +.L_byte1: ldrb ip, [r1], #1 uxtb r3, r2, ror #BYTE1_OFFSET subs ip, r3, ip - bne fast_return - m_cbz reg=r3, label=fast_return + bne .L_fast_return + m_cbz reg=r3, label=.L_fast_return -byte2: +.L_byte2: ldrb ip, [r1], #1 uxtb r3, r2, ror #BYTE2_OFFSET subs ip, r3, ip - bne fast_return - m_cbz reg=r3, label=fast_return + bne .L_fast_return + m_cbz reg=r3, label=.L_fast_return -byte3: +.L_byte3: ldrb ip, [r1], #1 uxtb r3, r2, ror #BYTE3_OFFSET subs ip, r3, ip - bne fast_return - m_cbnz reg=r3, label=word_aligned_r0 + bne .L_fast_return + m_cbnz reg=r3, label=.L_word_aligned_r0 -fast_return: +.L_fast_return: mov r0, ip bx lr -word_aligned_r0: +.L_word_aligned_r0: init /* The first string is word-aligned. */ /* Is the second string word-aligned? */ ands ip, r1, #3 - bne strcmp_unaligned + bne .L_strcmp_unaligned -word_aligned: +.L_word_aligned: /* The strings are word-aligned. */ /* Is the first string double-word aligned? */ tst r0, #4 - beq doubleword_aligned_r0 + beq .L_doubleword_aligned_r0 /* If r0 is not double-word aligned yet, align it by loading and comparing the next word from each string. */ ldr r2, [r0], #4 ldr r4, [r1], #4 - magic_compare_and_branch w1=r2 w2=r4 label=return_24 + magic_compare_and_branch w1=r2 w2=r4 label=.L_return_24 -doubleword_aligned_r0: +.L_doubleword_aligned_r0: /* Get here when r0 is double-word aligned. */ /* Is r1 doubleword_aligned? */ tst r1, #4 - beq doubleword_aligned + beq .L_doubleword_aligned /* Get here when the strings to compare are word-aligned, r0 is double-word aligned, but r1 is not double-word aligned. */ @@ -278,9 +278,9 @@ doubleword_aligned_r0: /* Load the next double-word from each string and compare. */ ldrd r2, r3, [r0], #8 - magic_compare_and_branch w1=r2 w2=r5 label=return_25 + magic_compare_and_branch w1=r2 w2=r5 label=.L_return_25 ldrd r4, r5, [r1], #8 - magic_compare_and_branch w1=r3 w2=r4 label=return_34 + magic_compare_and_branch w1=r3 w2=r4 label=.L_return_34 b 3b .macro miscmp_word offsetlo offsethi @@ -304,33 +304,33 @@ doubleword_aligned_r0: and r2, r3, r6, S2LOMEM #\offsetlo it eq cmpeq r2, r5 - bne return_25 + bne .L_return_25 ldr r5, [r1], #4 cmp ip, #0 eor r3, r2, r3 S2HIMEM r2, r5, #\offsethi it eq cmpeq r3, r2 - bne return_32 + bne .L_return_32 b 7b .endm /* miscmp_word */ -return_32: +.L_return_32: setup_return w1=r3, w2=r2 - b do_return -return_34: + b .L_do_return +.L_return_34: setup_return w1=r3, w2=r4 - b do_return -return_25: + b .L_do_return +.L_return_25: setup_return w1=r2, w2=r5 - b do_return -return_35: + b .L_do_return +.L_return_35: setup_return w1=r3, w2=r5 - b do_return -return_24: + b .L_do_return +.L_return_24: setup_return w1=r2, w2=r4 -do_return: +.L_do_return: #ifdef __ARMEB__ mov r0, ip @@ -341,7 +341,6 @@ do_return: /* Restore temporaries early, before computing the return value. */ ldrd r6, r7, [sp] ldrd r4, r5, [sp, #8] - .pad #-16 adds sp, sp, #16 .cfi_def_cfa_offset 0 .cfi_restore r4 @@ -352,7 +351,7 @@ do_return: /* There is a zero or a different byte between r1 and r2. */ /* r0 contains a mask of all-zero bytes in r1. */ /* Using r0 and not ip here because cbz requires low register. */ - m_cbz reg=r0, label=compute_return_value + m_cbz reg=r0, label=.L_compute_return_value clz r0, r0 /* r0 contains the number of bits on the left of the first all-zero byte in r1. */ rsb r0, r0, #24 @@ -360,7 +359,7 @@ do_return: lsr r1, r1, r0 lsr r2, r2, r0 -compute_return_value: +.L_compute_return_value: movs r0, #1 cmp r1, r2 /* The return value is computed as follows. @@ -380,7 +379,7 @@ compute_return_value: * bionic/libc/arch-arm/cortex-a15/bionic/strcmp.S for the unedited * version of the code. */ -strcmp_unaligned: +.L_strcmp_unaligned: wp1 .req r0 wp2 .req r1 b1 .req r2 @@ -532,7 +531,6 @@ strcmp_unaligned: /* Restore registers and stack. */ ldrd r6, r7, [sp] ldrd r4, r5, [sp, #8] - .pad #-16 adds sp, sp, #16 .cfi_def_cfa_offset 0 .cfi_restore r4 @@ -557,13 +555,7 @@ strcmp_unaligned: /* Restore registers and stack. */ ldrd r6, r7, [sp] ldrd r4, r5, [sp, #8] - .pad #-16 adds sp, sp, #16 - .cfi_def_cfa_offset 0 - .cfi_restore r4 - .cfi_restore r5 - .cfi_restore r6 - .cfi_restore r7 bx lr .cfi_endproc diff --git a/libc/arch-arm/krait/bionic/__strcat_chk.S b/libc/arch-arm/krait/bionic/__strcat_chk.S index 4b125c89c..ec99077c3 100644 --- a/libc/arch-arm/krait/bionic/__strcat_chk.S +++ b/libc/arch-arm/krait/bionic/__strcat_chk.S @@ -40,13 +40,13 @@ ENTRY(__strcat_chk) .cfi_startproc pld [r0, #0] - .save {r0, lr} push {r0, lr} + .save {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 .cfi_rel_offset lr, 4 - .save {r4, r5} push {r4, r5} + .save {r4, r5} .cfi_adjust_cfa_offset 8 .cfi_rel_offset r4, 0 .cfi_rel_offset r5, 4 @@ -180,22 +180,29 @@ ENTRY(__strcat_chk) .L_strlen_done: add r2, r3, r4 cmp r2, lr - bgt .L_fortify_check_failed + bgt __strcat_chk_failed // Set up the registers for the memcpy code. mov r1, r5 pld [r1, #64] mov r2, r4 add r0, r0, r3 - .pad #-8 pop {r4, r5} - .cfi_adjust_cfa_offset -8 - .cfi_restore r4 - .cfi_restore r5 - #include "memcpy_base.S" + .cfi_endproc +END(__strcat_chk) -.L_fortify_check_failed: +#define MEMCPY_BASE __strcat_chk_memcpy_base +#define MEMCPY_BASE_ALIGNED __strcat_chk_memcpy_base_aligned +#include "memcpy_base.S" + +ENTRY(__strcat_chk_failed) + .cfi_startproc + .save {r0, lr} + .save {r4, r5} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 .cfi_adjust_cfa_offset 8 .cfi_rel_offset r4, 0 .cfi_rel_offset r5, 4 @@ -211,7 +218,7 @@ error_message: .word error_string-(1b+4) .cfi_endproc -END(__strcat_chk) +END(__strcat_chk_failed) .data error_string: diff --git a/libc/arch-arm/krait/bionic/__strcpy_chk.S b/libc/arch-arm/krait/bionic/__strcpy_chk.S index d5e1db610..7da4d1536 100644 --- a/libc/arch-arm/krait/bionic/__strcpy_chk.S +++ b/libc/arch-arm/krait/bionic/__strcpy_chk.S @@ -39,8 +39,8 @@ ENTRY(__strcpy_chk) .cfi_startproc pld [r0, #0] - .save {r0, lr} push {r0, lr} + .save {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 .cfi_rel_offset lr, 4 @@ -151,14 +151,25 @@ ENTRY(__strcpy_chk) pld [r1, #64] ldr r0, [sp] cmp r3, lr - bge .L_fortify_check_failed + bge __strcpy_chk_failed // Add 1 for copy length to get the string terminator. add r2, r3, #1 - #include "memcpy_base.S" + .cfi_endproc +END(__strcpy_chk) + +#define MEMCPY_BASE __strcpy_chk_memcpy_base +#define MEMCPY_BASE_ALIGNED __strcpy_chk_memcpy_base_aligned +#include "memcpy_base.S" + +ENTRY(__strcpy_chk_failed) + .cfi_startproc + .save {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 -.L_fortify_check_failed: ldr r0, error_message ldr r1, error_code 1: @@ -169,7 +180,7 @@ error_code: error_message: .word error_string-(1b+4) .cfi_endproc -END(__strcpy_chk) +END(__strcpy_chk_failed) .data error_string: diff --git a/libc/arch-arm/krait/bionic/memcpy.S b/libc/arch-arm/krait/bionic/memcpy.S index 093a2443c..907240853 100644 --- a/libc/arch-arm/krait/bionic/memcpy.S +++ b/libc/arch-arm/krait/bionic/memcpy.S @@ -55,22 +55,23 @@ END(__memcpy_chk) ENTRY(memcpy) .cfi_startproc pld [r1, #64] - .save {r0, lr} stmfd sp!, {r0, lr} + .save {r0, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r0, 0 .cfi_rel_offset lr, 4 - - #include "memcpy_base.S" .cfi_endproc END(memcpy) - .fnstart +#define MEMCPY_BASE __memcpy_base +#define MEMCPY_BASE_ALIGNED __memcpy_base_aligned +#include "memcpy_base.S" + +ENTRY(__memcpy_chk_fail) .cfi_startproc -__memcpy_chk_fail: // Preserve lr for backtrace. - .save {lr} push {lr} + .save {lr} .cfi_def_cfa_offset 4 .cfi_rel_offset lr, 0 @@ -84,7 +85,7 @@ error_code: error_message: .word error_string-(1b+4) .cfi_endproc - .fnend +END(__memcpy_chk_fail) .data error_string: diff --git a/libc/arch-arm/krait/bionic/memcpy_base.S b/libc/arch-arm/krait/bionic/memcpy_base.S index 48ce477f0..d87a5423b 100644 --- a/libc/arch-arm/krait/bionic/memcpy_base.S +++ b/libc/arch-arm/krait/bionic/memcpy_base.S @@ -35,6 +35,13 @@ // Assumes neon instructions and a cache line size of 32 bytes. +ENTRY(MEMCPY_BASE) + .cfi_startproc + .save {r0, lr} + .cfi_def_cfa_offset 8 + .cfi_rel_offset r0, 0 + .cfi_rel_offset lr, 4 + /* do we have at least 16-bytes to copy (needed for alignment below) */ cmp r2, #16 blo 5f @@ -115,3 +122,6 @@ ldmfd sp!, {r0, lr} bx lr + + .cfi_endproc +END(MEMCPY_BASE) diff --git a/libc/arch-arm/krait/bionic/strcmp.S b/libc/arch-arm/krait/bionic/strcmp.S index f26aaf14c..d4cf3f42e 100644 --- a/libc/arch-arm/krait/bionic/strcmp.S +++ b/libc/arch-arm/krait/bionic/strcmp.S @@ -122,7 +122,6 @@ ENTRY(strcmp) .macro init /* Macro to save temporary registers and prepare magic values. */ - .save {r4-r7} subs sp, sp, #16 .cfi_def_cfa_offset 16 strd r4, r5, [sp, #8] @@ -178,12 +177,13 @@ ENTRY(strcmp) /* Are both strings double-word aligned? */ orr ip, r0, r1 tst ip, #7 - bne do_align + bne .L_do_align /* Fast path. */ + .save {r4-r7} init -doubleword_aligned: +.L_doubleword_aligned: /* Get here when the strings to compare are double-word aligned. */ /* Compare two words in every iteration. */ @@ -196,14 +196,14 @@ doubleword_aligned: ldrd r2, r3, [r0], #8 ldrd r4, r5, [r1], #8 - magic_compare_and_branch w1=r2, w2=r4, label=return_24 - magic_compare_and_branch w1=r3, w2=r5, label=return_35 + magic_compare_and_branch w1=r2, w2=r4, label=.L_return_24 + magic_compare_and_branch w1=r3, w2=r5, label=.L_return_35 b 2b -do_align: +.L_do_align: /* Is the first string word-aligned? */ ands ip, r0, #3 - beq word_aligned_r0 + beq .L_word_aligned_r0 /* Fast compare byte by byte until the first string is word-aligned. */ /* The offset of r0 from a word boundary is in ip. Thus, the number of bytes @@ -211,58 +211,58 @@ do_align: bic r0, r0, #3 ldr r2, [r0], #4 lsls ip, ip, #31 - beq byte2 - bcs byte3 + beq .L_byte2 + bcs .L_byte3 -byte1: +.L_byte1: ldrb ip, [r1], #1 uxtb r3, r2, ror #BYTE1_OFFSET subs ip, r3, ip - bne fast_return - m_cbz reg=r3, label=fast_return + bne .L_fast_return + m_cbz reg=r3, label=.L_fast_return -byte2: +.L_byte2: ldrb ip, [r1], #1 uxtb r3, r2, ror #BYTE2_OFFSET subs ip, r3, ip - bne fast_return - m_cbz reg=r3, label=fast_return + bne .L_fast_return + m_cbz reg=r3, label=.L_fast_return -byte3: +.L_byte3: ldrb ip, [r1], #1 uxtb r3, r2, ror #BYTE3_OFFSET subs ip, r3, ip - bne fast_return - m_cbnz reg=r3, label=word_aligned_r0 + bne .L_fast_return + m_cbnz reg=r3, label=.L_word_aligned_r0 -fast_return: +.L_fast_return: mov r0, ip bx lr -word_aligned_r0: +.L_word_aligned_r0: init /* The first string is word-aligned. */ /* Is the second string word-aligned? */ ands ip, r1, #3 - bne strcmp_unaligned + bne .L_strcmp_unaligned -word_aligned: +.L_word_aligned: /* The strings are word-aligned. */ /* Is the first string double-word aligned? */ tst r0, #4 - beq doubleword_aligned_r0 + beq .L_doubleword_aligned_r0 /* If r0 is not double-word aligned yet, align it by loading and comparing the next word from each string. */ ldr r2, [r0], #4 ldr r4, [r1], #4 - magic_compare_and_branch w1=r2 w2=r4 label=return_24 + magic_compare_and_branch w1=r2 w2=r4 label=.L_return_24 -doubleword_aligned_r0: +.L_doubleword_aligned_r0: /* Get here when r0 is double-word aligned. */ /* Is r1 doubleword_aligned? */ tst r1, #4 - beq doubleword_aligned + beq .L_doubleword_aligned /* Get here when the strings to compare are word-aligned, r0 is double-word aligned, but r1 is not double-word aligned. */ @@ -278,9 +278,9 @@ doubleword_aligned_r0: /* Load the next double-word from each string and compare. */ ldrd r2, r3, [r0], #8 - magic_compare_and_branch w1=r2 w2=r5 label=return_25 + magic_compare_and_branch w1=r2 w2=r5 label=.L_return_25 ldrd r4, r5, [r1], #8 - magic_compare_and_branch w1=r3 w2=r4 label=return_34 + magic_compare_and_branch w1=r3 w2=r4 label=.L_return_34 b 3b .macro miscmp_word offsetlo offsethi @@ -304,46 +304,46 @@ doubleword_aligned_r0: and r2, r3, r6, S2LOMEM #\offsetlo it eq cmpeq r2, r5 - bne return_25 + bne .L_return_25 ldr r5, [r1], #4 cmp ip, #0 eor r3, r2, r3 S2HIMEM r2, r5, #\offsethi it eq cmpeq r3, r2 - bne return_32 + bne .L_return_32 b 7b .endm /* miscmp_word */ -strcmp_unaligned: +.L_strcmp_unaligned: /* r0 is word-aligned, r1 is at offset ip from a word. */ /* Align r1 to the (previous) word-boundary. */ bic r1, r1, #3 /* Unaligned comparison word by word using LDRs. */ cmp ip, #2 - beq miscmp_word_16 /* If ip == 2. */ - bge miscmp_word_24 /* If ip == 3. */ + beq .L_miscmp_word_16 /* If ip == 2. */ + bge .L_miscmp_word_24 /* If ip == 3. */ miscmp_word offsetlo=8 offsethi=24 /* If ip == 1. */ -miscmp_word_24: miscmp_word offsetlo=24 offsethi=8 +.L_miscmp_word_24: miscmp_word offsetlo=24 offsethi=8 -return_32: +.L_return_32: setup_return w1=r3, w2=r2 - b do_return -return_34: + b .L_do_return +.L_return_34: setup_return w1=r3, w2=r4 - b do_return -return_25: + b .L_do_return +.L_return_25: setup_return w1=r2, w2=r5 - b do_return -return_35: + b .L_do_return +.L_return_35: setup_return w1=r3, w2=r5 - b do_return -return_24: + b .L_do_return +.L_return_24: setup_return w1=r2, w2=r4 -do_return: +.L_do_return: #ifdef __ARMEB__ mov r0, ip @@ -354,7 +354,6 @@ do_return: /* Restore temporaries early, before computing the return value. */ ldrd r6, r7, [sp] ldrd r4, r5, [sp, #8] - .pad #-16 adds sp, sp, #16 .cfi_def_cfa_offset 0 .cfi_restore r4 @@ -365,7 +364,7 @@ do_return: /* There is a zero or a different byte between r1 and r2. */ /* r0 contains a mask of all-zero bytes in r1. */ /* Using r0 and not ip here because cbz requires low register. */ - m_cbz reg=r0, label=compute_return_value + m_cbz reg=r0, label=.L_compute_return_value clz r0, r0 /* r0 contains the number of bits on the left of the first all-zero byte in r1. */ rsb r0, r0, #24 @@ -373,7 +372,7 @@ do_return: lsr r1, r1, r0 lsr r2, r2, r0 -compute_return_value: +.L_compute_return_value: movs r0, #1 cmp r1, r2 /* The return value is computed as follows. @@ -393,7 +392,7 @@ compute_return_value: * previous version. See bionic/libc/arch-arm/cortex-a15/bionic/strcmp.S * for the unedited version of this code. */ -miscmp_word_16: +.L_miscmp_word_16: wp1 .req r0 wp2 .req r1 b1 .req r2 @@ -465,7 +464,6 @@ miscmp_word_16: /* Restore registers and stack. */ ldrd r6, r7, [sp] ldrd r4, r5, [sp, #8] - .pad #-16 adds sp, sp, #16 .cfi_def_cfa_offset 0 .cfi_restore r4 @@ -490,7 +488,6 @@ miscmp_word_16: /* Restore registers and stack. */ ldrd r6, r7, [sp] ldrd r4, r5, [sp, #8] - .pad #-16 adds sp, sp, #16 .cfi_def_cfa_offset 0 .cfi_restore r4 From 883ef2499c2ff76605f73b1240f719ca6282e554 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Tue, 10 Sep 2013 16:56:34 -0700 Subject: [PATCH 64/77] __memcpy_chk: Fix signed cmp of unsigned values. I accidentally did a signed comparison of the size_t values passed in for three of the _chk functions. Changing them to unsigned compares. Add three new tests to verify this failure is fixed. Bug: 10691831 Change-Id: Ia831071f7dffd5972a748d888dd506c7cc7ddba3 --- .../arch-arm/cortex-a15/bionic/__strcat_chk.S | 2 +- .../arch-arm/cortex-a15/bionic/__strcpy_chk.S | 2 +- libc/arch-arm/cortex-a15/bionic/memcpy.S | 2 +- libc/arch-arm/cortex-a9/bionic/__strcat_chk.S | 2 +- libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S | 2 +- libc/arch-arm/cortex-a9/bionic/memcpy.S | 2 +- libc/arch-arm/krait/bionic/__strcat_chk.S | 2 +- libc/arch-arm/krait/bionic/__strcpy_chk.S | 2 +- libc/arch-arm/krait/bionic/memcpy.S | 2 +- tests/fortify_test.cpp | 54 +++++++++++++++++++ 10 files changed, 63 insertions(+), 9 deletions(-) diff --git a/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S b/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S index 46936005d..4aaa9f1f8 100644 --- a/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S +++ b/libc/arch-arm/cortex-a15/bionic/__strcat_chk.S @@ -180,7 +180,7 @@ ENTRY(__strcat_chk) .L_strlen_done: add r2, r3, r4 cmp r2, lr - bgt __strcat_chk_failed + bhi __strcat_chk_failed // Set up the registers for the memcpy code. mov r1, r5 diff --git a/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S b/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S index 1224b491d..05152e63d 100644 --- a/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S +++ b/libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S @@ -151,7 +151,7 @@ ENTRY(__strcpy_chk) pld [r1, #64] ldr r0, [sp] cmp r3, lr - bge __strcpy_chk_failed + bhs __strcpy_chk_failed // Add 1 for copy length to get the string terminator. add r2, r3, #1 diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy.S b/libc/arch-arm/cortex-a15/bionic/memcpy.S index a300e43db..a8432306e 100644 --- a/libc/arch-arm/cortex-a15/bionic/memcpy.S +++ b/libc/arch-arm/cortex-a15/bionic/memcpy.S @@ -65,7 +65,7 @@ ENTRY(__memcpy_chk) .cfi_startproc cmp r2, r3 - bgt __memcpy_chk_fail + bhi __memcpy_chk_fail // Fall through to memcpy... .cfi_endproc diff --git a/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S b/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S index cc43456c3..78cf19a2e 100644 --- a/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S +++ b/libc/arch-arm/cortex-a9/bionic/__strcat_chk.S @@ -183,7 +183,7 @@ ENTRY(__strcat_chk) .L_strlen_done: add r2, r3, r4 cmp r2, lr - bgt __strcat_chk_fail + bhi __strcat_chk_fail // Set up the registers for the memcpy code. mov r1, r5 diff --git a/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S b/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S index dd3370bae..d0acf1e4f 100644 --- a/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S +++ b/libc/arch-arm/cortex-a9/bionic/__strcpy_chk.S @@ -153,7 +153,7 @@ ENTRY(__strcpy_chk) pld [r1, #64] ldr r0, [sp] cmp r3, lr - bge __strcpy_chk_fail + bhs __strcpy_chk_fail // Add 1 for copy length to get the string terminator. add r2, r3, #1 diff --git a/libc/arch-arm/cortex-a9/bionic/memcpy.S b/libc/arch-arm/cortex-a9/bionic/memcpy.S index 21e0ebea6..5c4c42821 100644 --- a/libc/arch-arm/cortex-a9/bionic/memcpy.S +++ b/libc/arch-arm/cortex-a9/bionic/memcpy.S @@ -43,7 +43,7 @@ ENTRY(__memcpy_chk) .cfi_startproc cmp r2, r3 - bgt __memcpy_chk_fail + bhi __memcpy_chk_fail // Fall through to memcpy... .cfi_endproc diff --git a/libc/arch-arm/krait/bionic/__strcat_chk.S b/libc/arch-arm/krait/bionic/__strcat_chk.S index ec99077c3..956b4615a 100644 --- a/libc/arch-arm/krait/bionic/__strcat_chk.S +++ b/libc/arch-arm/krait/bionic/__strcat_chk.S @@ -180,7 +180,7 @@ ENTRY(__strcat_chk) .L_strlen_done: add r2, r3, r4 cmp r2, lr - bgt __strcat_chk_failed + bhi __strcat_chk_failed // Set up the registers for the memcpy code. mov r1, r5 diff --git a/libc/arch-arm/krait/bionic/__strcpy_chk.S b/libc/arch-arm/krait/bionic/__strcpy_chk.S index 7da4d1536..402cac678 100644 --- a/libc/arch-arm/krait/bionic/__strcpy_chk.S +++ b/libc/arch-arm/krait/bionic/__strcpy_chk.S @@ -151,7 +151,7 @@ ENTRY(__strcpy_chk) pld [r1, #64] ldr r0, [sp] cmp r3, lr - bge __strcpy_chk_failed + bhs __strcpy_chk_failed // Add 1 for copy length to get the string terminator. add r2, r3, #1 diff --git a/libc/arch-arm/krait/bionic/memcpy.S b/libc/arch-arm/krait/bionic/memcpy.S index 907240853..c69d8902d 100644 --- a/libc/arch-arm/krait/bionic/memcpy.S +++ b/libc/arch-arm/krait/bionic/memcpy.S @@ -46,7 +46,7 @@ ENTRY(__memcpy_chk) .cfi_startproc cmp r2, r3 - bgt __memcpy_chk_fail + bhi __memcpy_chk_fail // Fall through to memcpy... .cfi_endproc diff --git a/tests/fortify_test.cpp b/tests/fortify_test.cpp index d8f0e7619..85ce270d4 100644 --- a/tests/fortify_test.cpp +++ b/tests/fortify_test.cpp @@ -657,3 +657,57 @@ TEST(TEST_NAME, strcat2) { ASSERT_EQ('7', buf[8]); ASSERT_EQ('\0', buf[9]); } + +TEST(TEST_NAME, strcat_chk_max_int_size) { + char buf[10]; + memset(buf, 'A', sizeof(buf)); + buf[0] = 'a'; + buf[1] = '\0'; + char* res = __strcat_chk(buf, "01234567", (size_t)-1); + ASSERT_EQ(buf, res); + ASSERT_EQ('a', buf[0]); + ASSERT_EQ('0', buf[1]); + ASSERT_EQ('1', buf[2]); + ASSERT_EQ('2', buf[3]); + ASSERT_EQ('3', buf[4]); + ASSERT_EQ('4', buf[5]); + ASSERT_EQ('5', buf[6]); + ASSERT_EQ('6', buf[7]); + ASSERT_EQ('7', buf[8]); + ASSERT_EQ('\0', buf[9]); +} + +extern "C" char* __strcpy_chk(char*, const char*, size_t); + +TEST(TEST_NAME, strcpy_chk_max_int_size) { + char buf[10]; + char* res = __strcpy_chk(buf, "012345678", (size_t)-1); + ASSERT_EQ(buf, res); + ASSERT_EQ('0', buf[0]); + ASSERT_EQ('1', buf[1]); + ASSERT_EQ('2', buf[2]); + ASSERT_EQ('3', buf[3]); + ASSERT_EQ('4', buf[4]); + ASSERT_EQ('5', buf[5]); + ASSERT_EQ('6', buf[6]); + ASSERT_EQ('7', buf[7]); + ASSERT_EQ('8', buf[8]); + ASSERT_EQ('\0', buf[9]); +} +extern "C" void* __memcpy_chk(void*, const void*, size_t, size_t); + +TEST(TEST_NAME, memcpy_chk_max_int_size) { + char buf[10]; + void* res = __memcpy_chk(buf, "012345678", sizeof(buf), (size_t)-1); + ASSERT_EQ((void*)buf, res); + ASSERT_EQ('0', buf[0]); + ASSERT_EQ('1', buf[1]); + ASSERT_EQ('2', buf[2]); + ASSERT_EQ('3', buf[3]); + ASSERT_EQ('4', buf[4]); + ASSERT_EQ('5', buf[5]); + ASSERT_EQ('6', buf[6]); + ASSERT_EQ('7', buf[7]); + ASSERT_EQ('8', buf[8]); + ASSERT_EQ('\0', buf[9]); +} From 9a74e36f4810ca22cb12f8eafe4f929edfa57e58 Mon Sep 17 00:00:00 2001 From: Brian Carlstrom Date: Thu, 12 Sep 2013 21:47:20 -0700 Subject: [PATCH 65/77] Use kernel default for initial thread size Bug: 10697851 Change-Id: I8d980f5e0b584799536f6e6b891056c968d26cdf --- libc/bionic/libc_init_common.cpp | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/libc/bionic/libc_init_common.cpp b/libc/bionic/libc_init_common.cpp index 1fc490e23..c10abadf7 100644 --- a/libc/bionic/libc_init_common.cpp +++ b/libc/bionic/libc_init_common.cpp @@ -37,6 +37,8 @@ #include #include #include +#include +#include #include #include "atexit.h" @@ -62,6 +64,21 @@ uintptr_t __stack_chk_guard = 0; unsigned int __page_size = PAGE_SIZE; unsigned int __page_shift = PAGE_SHIFT; +static size_t get_stack_size() { + const size_t minimal_stack_size = 128 * 1024; + size_t stack_size = minimal_stack_size; + struct rlimit stack_limit; + int rlimit_result = getrlimit(RLIMIT_STACK, &stack_limit); + if ((rlimit_result == 0) && (stack_limit.rlim_cur != RLIM_INFINITY)) { + stack_size = stack_limit.rlim_cur; + stack_size = (stack_size & ~(PAGE_SIZE - 1)); + if (stack_size < minimal_stack_size) { + stack_size = minimal_stack_size; + } + } + return stack_size; +} + /* Init TLS for the initial thread. Called by the linker _before_ libc is mapped * in memory. Beware: all writes to libc globals from this function will * apply to linker-private copies and will not be visible from libc later on. @@ -76,9 +93,9 @@ unsigned int __page_shift = PAGE_SHIFT; void __libc_init_tls(KernelArgumentBlock& args) { __libc_auxv = args.auxv; - unsigned stack_top = (__get_sp() & ~(PAGE_SIZE - 1)) + PAGE_SIZE; - unsigned stack_size = 128 * 1024; - unsigned stack_bottom = stack_top - stack_size; + uintptr_t stack_top = (__get_sp() & ~(PAGE_SIZE - 1)) + PAGE_SIZE; + size_t stack_size = get_stack_size(); + uintptr_t stack_bottom = stack_top - stack_size; static void* tls[BIONIC_TLS_SLOTS]; static pthread_internal_t thread; From cc7f0dc5a1806613fc9af4df484a34d8dd4eb06d Mon Sep 17 00:00:00 2001 From: Elliott Hughes Date: Fri, 13 Sep 2013 09:43:15 -0700 Subject: [PATCH 66/77] Upgrade pre-jb-mr2 releases to tzdata2013d. From the release notes: Changes affecting future time stamps: Morocco's midsummer transitions this year are July 7 and August 10, not July 9 and August 8. (Thanks to Andrew Paprocki.) Israel now falls back on the last Sunday of October. (Thanks to Ephraim Silverberg.) Changes affecting past time stamps: Specify Jerusalem's location more precisely; this changes the pre-1880 times by 2 s. Changing affecting metadata only: Fix typos in the entries for country codes BQ and SX. Change-Id: I8caa139f0eecbe038643df43a27088278c6185cd --- libc/zoneinfo/zoneinfo.dat | Bin 484319 -> 518429 bytes libc/zoneinfo/zoneinfo.idx | Bin 29952 -> 30108 bytes libc/zoneinfo/zoneinfo.version | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/zoneinfo/zoneinfo.dat b/libc/zoneinfo/zoneinfo.dat index cc1cb389d7fd5a7ec88b5948c6c873945b1923b3..97b79b8053e1be0bf043f5a33ea26a72d66e064f 100644 GIT binary patch delta 51840 zcmeFa30O^U8~?l3+PjiOBtpm-Qtb?xlAX*%k`&TTl0vqSLe>r$lN7d)$~@bIjnFpF zWXxq=}UO(Ucxu1JIYp?sc*W-TOvL2qUopH6U zu5CqrYfih0edYW>Ex+2Nrnb1e+)CRiL~fF%uj0T5r#-9^Czi|2@^M{Z?-RZ;tN*ka zS^aOF&+;`{o#oqoUeJ2?4a)oW#5;SzB814Z&Ld1Q1<;v=^mi$KA`MgNa=n^>7JnMzM$;h(jMB_ zlxO!v%gDTRQPqCiTK zAf;%KQbb5|22}InDJ(h5ss~rnYJTulohr`gc6rpLHXEbz;m39E&1Bnb0!?u zOu?HE@EqXBd3n-*+AFi=_F5x5&$mhmJ5+Vmtw?DOQksWUnoA~doNXVPj7=%Wl;)Vy zvX8eP_g~Yp`Pq@?+N8l^l}*kChpE+gpR^tx(?p+)bgsYOZH;fl9?gJOxf(y;E8M`U z85;i&QDHm{m6&Lt%GB^0}bZ*GiKyGM;iVND^fg2XznG0SyNTUjB$c0R8 zry1VSlnWhFOEaQzeN9;V-QJXpS%vs#{z6-eM(2?8(=ds+xJ~K3vD)!@Invc-LymRGZD|FMu7Iw2$E71Nhy+07EMx$D3nE&lp;$9MVFK!Ovl({@iabAr0Jk& z(?JoZgQ89cMV=0dJ{`0Iq_hfj&`LnrYS2L|0%fa0N-IN3t3ygFL^`b&TPFHIOGO7Q z7wP}mlDXC_@72}M$)JRimE99(JCfe~M(i#R` z(7!7d6nt8}VLkYfIg!Is_?&WUDhiAnrTM5#()z4(!3l9H_ts+HF$!NFqjG?AUxlCI zm~vo6g~H!zoznkXAKjp$*~)+dXI)_aDCJ|M&Sa|3lz32Dk6<3@GRsUvq9#`mbLi|)19Fu|cjH?i3S!=%b~ikR{P4KYQ36q8@I zF~sH@6>;Zk7^disDW>MQDW|2ZQ^cp)DyOSwE5zuZN^NB;VUSi~B{a<`UUpNU`=V&l zy^+%WkGWr3`vbE_Pe zRpC6}F-@CMoL0F?$F$3tJ@}3}wpm5|KS-HbJI%1cCMRWG4O5e+`x8T-rtR3`WLAsg zN=nYuN?X3QvG&j^d4sf!E#_9V*Kdw%_>>f$UAB7v$1`X7)>r0M>SnSDK8{C%lmai5 zlpIV+OWo^dUPS)O0|u(o-tVn$mO?WhOv$m_SF<(moz~=abG~KTz}J4}-sHCSMK$lV z-LJjPT9VtYmN}+x^2=E?_`3W}51KJADDBbb0hJn%>;J}PpZ0TT?HtXQXKe5N>eJud z+Q)a&EuR6kzWDeJJm};1tH8&<^(r6#!cX}Db@t~6oVbx6_-94_;C1@^ArGeI51o~s zA9Q3`{;&~K^Mh9_^HtuTo;0>TWWX97au(`n?Ok~%ZNGSayUf+uRePnp%ucKDl2_HcOy>(P^7HjB zodh32`!tB3%&M_E@{$sLjP`e5VE`NI-%n_3!UOH~Tl|E3efdG!IctSg+T|MljJ71( zv~Jq2wQaRM)(HW8fIfSj(3aO*t>ja9?U>jf+Aoi!#E>67s^iX1~`gQz_K$3zXwyIxC_Z zZ8S{qZ>X4PGS4u{%~Ubz>1e}bn-_}7C%p}^U(PAw*1H&{Jb0#?npoK|?Z_Eje3Y|t z`syt@v9Fa<-|d!Ak>?lc2i_JY$>_Fd8R*tX>GnwJ7D?$gq3l*k>2{&)mPzThN$J)} z>Gnw}1f&!OQVInrg@cqrLMl>R=mW)ul;T55F(RcnL0POwDPB+(Gg68hl*Nve;zvp` zB&9f#QY=X+o}?60ChWHw7FRM*Y)L7;q!eRPiZdz2nv~)VWicnExIY@iPeOMc;t&|Sd9w`AYe?JwRzWNb(#jWKEjvXuK{5H__ zxmEk8z>pFC&_KL;`A~3X_oLvfSPyWv!&>l~0DJISvqbPZPj?+Vja&CT9K4}k9q@+y zSxT~R;yLgp-AM3e<74oalpZ1sTMZ{**rsj<-k!Aqyu+sgc;~$13XUB*mv`V@5$nOb zYm_&TJ*Vd!aL)HP;Jr;Fzk6#?K&Jr9E~&zz{^*pc7jj*|3Bs6Y5> zl|$fbUTwkGKCTAeXiyV;!nf@8<+)GpvV=YN6kPE{GmesAl2 z@ckj3!4DSXfgg5g2sVz+20yB03Vz&sCiwBs7vREHnL3Ugg?G+@pH)pD7P)7~pDWo1 z?)m3XIJ{VU7X0#hSMaNuTfwjQIDp@TE(E{L=%Qe!aqqmwf#1hB1{XD7U?BU4pmN}k z6~=-;xxWH`deaa5#qI)Qmj7QaxWe$Ycn2K5ZL77l4b` zCxd_ceO8kFPof%J(&{EL&xQFC^L*9)V4-U(u<&^WSnk|kLD%Qy*Qdc?TCt^qPB1;Z z+(2sf&IVjAV+z>(;x};l_@Q8n?YF@df;_;M3vl@i5p_i-&{F$tU1yUxUHb*PjI2-t+|5NZbgnnKw+qvGZ$19XF8L zWh%k7`>qGq5$l8NI?n;y2g||rDn@|odpr|isQ<1v3=NwII*v`$@RAd_u~{LwvA&m* z><-UQftzTXgPWe-1a3CDBG_@`d~kFBBCyk(F<@u6t6&%PJO#(*QB>OlcJ&cQ!{Azy zZ=ive3U6@Bhr7Y8tz5vZ^Ok_!iYkNKWX6Eo-geT_dF{jiaQg$5z#T?SQj*<$`3G>v z9)rLhvDd(znzREe#elsqc$TXLL+7?yaOda0!CmU@QE+UcE~g)YyYb7w-E_YUWbghc z7ThD{KDg)6!Qfu%JaF%9cW@t{Y_NAi9dKX8Ot9E5vb~OD6ZzE0Qj+%XWe4v6V+MFY z(?8$=w?}{nnj66b4|D_jzdQyWl->j!aP^^rWAg;Yl?M;reH1)ocw+@r4Ong6-$3^9UYEe*KeYr; zXrKd6xMl;MBufQP+Wie2^JF4;a_Vhx?1dB^$L5KP&IM1|h7ymTI^>oT&;PU}KRCp9 zIH)AY>0?mi@nWr2U`-#?c)aEpYCL|1<4^d{xU0hWEF08#{Hznx40L|t=O6e2v)2s+ z&$)gVJa?87Ja5lo@ca?=4S4<+EXjl+sf!#A3*&QjG*NPcXW&Ia8Q>I4nUd^_JBr}N zZ&2g$X|+SZX%|joe93Rrc>L0Bo59QOH^R6!2_+uC{8VQJ&67T6tO$n{8ydo}av*9v ze${MKa7OFV269{-_5z$~;|LXF4kOi|4Eq0a!22c8A0MyiYnh@xaJB;Debr zz`6ZUBjdk}~kkAGNB2R62I!npATN<98?jY{ChXC`92P;k`Y z`7hk$55v>PmdwEM&lXQml0HBB9{ghbK=8|K)Oh@>{;2Wz*9kcof8$yM<8LGL6?EP^ zt6$*vy-?%v?|)V|ko`lmB^dv3`v>?_c~;`F=a2t%;4TcGU!lh1zoZ`qe?8Y8Cwz;` z1b@##jmQ58OaT8(LygD(>WmtXFP=~nU+8yzjgn*Y{_#bP$Cn7hi3P630~!~&$6dfe z6$5_%7ldPtVUYd0PZt#ATRR&_P44D_O&2r-o8@MM%Z)Y#o3ES+F5e3^o?tOG3|yhv zIgDG5NYHU?o{HtSfh%=Ei6>P0bXH0B$_=|>BM~ZJLyad`%N=lnbRer^FyXz(Q&+;H7zCD|L@P=g!KLX9Ul>_v?yG#RlU<4u>g!g#YT zE5MGEs)3s~oCbCp)>1+9ILn8DT|^Jmc!JB@DFzyFt^ExSt`~=bTl~HaZn@n9-0FTV zxb;HRc!JyE3~-yV0>;~JIH03>+WFU2lD40{65PR!2fL4q2Y0l20v0{G1;gO+Wm4+!eHc{uj^T1y3QR4|+RNwH3D0mM-jVE~jse}{ywK=6@146&&sPTmU)i;6rpRTARyYDyDc*1}U zMPR>zF&G~>`zqMqFi$~#gGTNF2V{)~2lmT1kbSVm8$85?N<5JhhN@g(2&%LM4#SkF z@q}UTW5BAqpD?bvj2cfEZeB@86Aja7_{dn)ctZFllz76Z zfLcmAFJi71{Q)(eFd-`&Jn>2$j8D>_#uH+8ap1|SFpS5pK#3>BDZAtMe_=}8AAEtSO;O_s z(*lj)c=K*by1?{y$H3EHHUVpDtOIM#qQ(;ve#{2X*z^YDGas*2(0Q{`%Na-$k0pX< zkADT8vj!!eFt`5&5r%m)dctA8D{4Gp!Ei@#QsoqkFYMi2M-wgl5Di}B*c80z4r)AM zv4sVA@j=vhLh7rx;ItJ3lyu&bbCBK=#!wCxTZ$z75W@_6KJj%LQi_N9)LM&6*7G+IxQBbqfwE$-e%O z8+d~_8a1AtvaM(QcCmgmoqQ(=p4!H;3X3-hr+q?Ww&_vr`Hw5o&co)3$ zy3#=QUGl@=U3=<-cNb=YbC#gQ6ZV`)0E>GkJcGl&b)him2b=-#Unp(O2i%`3IX213pkt#^9G+@iW*Ni)71rhcG5PCpKF8~PdGm?8GJ$h3F8+%w&-Y{i-kA9 zm+K^fFJJtqBzyiJlz76G?bpFqA5e+Mp8snL_rY-eNDG{BV=QVs;pWDE3OcdCdkV&H z%|VSP+;&@LAp4zgYmDEmh8j<}*X;}V-q*nxf6%x9{NN@^Ji%ys0Bqb>2jh?4vl5S< z|HsQX7@l0#>uBOav9^-*>CSZUvtcFR=ZjF|2`@YzfnSab!uYE?$HA}rbppTsvmX4m zEowaB?en7sj?MSp)1gs<5g zF7U0pBgVgL9)o` z6E&WUe~B7TCM?=bC&*-HBNU|aSk!njlg+)srU5zw+0EuTfy=c`1)E1y0++W%jVH74 zb<~lc#dm*9Yu{Nv-)raTsgZ99I9MFjVH5Cr~$6JYbnNU zRKLL0R_s>LMAen|!M0OS3?{b3+`&A~AkuVsE0Y}Y;jZ1?goxQS`rvgX1;94Wm)x$r`OG1~>K{j&X+>sPSY?T66(79d5vQ zvnq|jj=k?II5v;t$8uoN$q_Z4%;_#_Jei9HYCM@suBm|>6|ZOF1lJWWz%9;)fm=>N zjVEihwL7@=VAOarwN+xt0yJIGMtHJ7=!GKaza$v8N8 zSic5)9PbbA^t+sbPE@XKslWb3NS5hG{1DW$YYb*aQ}&3NMTke9JKSIZyZ2evWcxTGxR$gR-vBpm&<9_jOc>|8nacO$qdQ>Jj$xq_}v*3|WpK$oRA?`v=yb?}vk3NV+Kd zl@9H>1X)A&_m=cK5@p}b|EK9~X~(9>zVf5>BNofTgtFFFS!=7TwN=*IDr;?(wYJJy zTV<`Sves5vYpblaRo2=nYi*Uaw#r&tWv#8U)>c_-tE{zE*4ip-ZI!jQ%351xt*!sr zT3b&4BdskZwNB!-io5*q|Lb4U6g8FFS1B@oy;di)5={MBPperoZOJrwH?y|?K%aPd z3-;rX6|@<9WafI$9NA)-*2RT4*WcYIv*Pui^s;umR#zZvt&LwNRA)cVYaz(O^m7i# znhW~lxw0j;)D@PwjY(0Fwn-QbZQCar+AeaAk)uvtb3bs0UJ+pT3U1(z&3c17e({BN zD(@uejaE?Qt4dJMz15(d&wennAGG}M|FIu1v(_4#upbwkab4ytOG{Z+LC?N#jt8Hj z?^s73$ct}mjXm_0^Lw8EY3!BduIjzrc~zf~!};Dzeyr--`j}6@iFZ{#LlztR`yE#K zcG;mC&?-~q*Dysjuxf(JUmmUU|D9nRROqJ)xGx$5Pq?WD9}Y1NS!b&nn&D*(TG+}s zEWVL3cx*MJD#*+j(&xKzc(;bC(B^lHBO05k!YUjxiX%;4s78LdtqOm7PBrRAt}5c> zHdW-_4AtoM$*M6+Mb+5kEyk!xA*yjvNk;WRFV*~AxHnNw83bF+KMqBw_L3m@U9ac+Ty{5a0V+Tcsl%hjQ5z;QICASe- zc*sB@BBd~qQm9BNT%;5-QVJWCg^rZMM@k_ir7)6GC`l=tq!dz83M(mvmQD3Q}4QQd$yHS{71T8Yb*#FxmRB5A3J+ zlZ>RaPNcL}q_ke7v}UBVZltt!P_}-gw1%X#j-<4fq_m!-w5FuAuB5cKbbXE%m<+VY zq*!RC8vg^}gl=1)GdkTcGBY=`aT0h|T|GF_Gzy$pk`A8zem->0!>Q1@m&ZWo?k;2 z6NXzSx`J}Gl)g*>6;xeJiC_(eSSgO@m?HC0KeQ84SlsL6#9B;vXNuQn`vFaZzrk2 z?}j>n-w*T!7j^Z5erPT2cpn=z0)Mip2K{Ve##H3E&!4};@TKr2^y>o~6~~TmC(eVv zAN~gZv2Hu~XU1*tuUU)0#qrXP_j~L%*#87cJ6?%*GBM9}M3VEolNu~EUTGw|%+eRn zKQA|phe7_f6&y^SVaMZ5Z~DM)b_zQlUv6Iuu=$1t(DKWwLM;|4RUDhJ!lbWY%W?I= z6$jn|SL!DRTeUt2uI&5_T%`{CU5#TWT30*+gY};S(5mmS~t8OLMj9^Wyf64;|ZHaxzQsQdtfvb~*=V-tGXUjuip@dxb1 z?FD=NFoL@nwcxI|kAb_L&4G45fE|zTv3V)DXF7H~zSq3T;NCOVsyH@(pNK%P_wYop z*tcf~82WV%hry>s0JML7?0CFymGicp@J(*Z{+@Ub~>d?bkq6Eib7!HepE3S>WN;I`HtH zBf+7cQo$qc^Z=9{gp^-D2LPt;YGt%?Vj|sJaVeC*h zW?(;o*7+@XoVzVp-P{~Hz7}>oKHB0nIQrK;=!Dl7p%V*#s5o~1r1Lw$F(>YVCvRO0 zj$L;c9JgQsc*?9y>G_{JF$DwDMqtO|t(7CI=L+2&EG>ROXVE(i_Ff52MRS_o*Jp^7j z@C7*8eKmMd>vQ0gM!C?%Hrt@7CK-}`Nrt996`@NWY%y|dzNIHaz{`##fwk+sz{^)h zNq3Z<*a(Id(|zEuGRhOWYM8X+W%O+TUhPp8n(1r;&2n&6aqI%wme0X!%&fs{-kt@o zef9;s?&dS-`cnna4g1bOH*OH69dFaJEii0ebi>H83Ae-~fwzv+gSYudfw%Wd2k&t6 z0q=B~3f9#f1Km|g+VOUk^abyJ-wB%YPyyX@xgKWq8#jPY zoDKquCk@h$cS;wBfzw%$&@(ARpl3CGpy$*bq30taR2;j&1)sX$i#>aTFDZEN<)%*H z{2FzjSIngy@5&DjeD#I2;a$7+m{{bmpSgyC8~aZ{Z|-_;WCMJ`@(tiyi?4uh$Ib=c z887X4cSF{}ey_i@rpRbU1ychE=!7t6lh(9h@dR2;K^**yvTb!#^GTk0V2_XRV- zKcd@#e~u0V{~F>CE$%Jtc)vTe1^;oBcD#~WHHihTYF8ulXR(T&e}Vt(0E2Mnfr?mm z9Xp;NKmHSTlN{`Lg6Z0OV6!yrctW|EN5JM2c0$XCW?07=UlV8ED^6INNc7fLGZ-L!1<={4n2f=NpKLfXm zDuA{hb_Uv^?*XX02R1xG?C87_29G*P&`uTOp-Qt&Mvh%T`7Rjj`FuXO^CeHP*Xc3f zF8Uy7*Nxcmgl^hS;O=u2&>oZPL3^sbRUDhI*B}|Vw~q_4DD-iA3WK+zG8u%v)lP%^ zRr&<(S5gS|d5;}W=>PB(*!Qv?Iv{To)NlVaBgf7km^mNpzkDxvkW}Ic0kK+e;HbIK z!2z`4v7bB{(jyXvp>2migPQb#4yzZd;@E`2<($DPelS?|!U`Pn$Q?ZV^ha>$(K_G} zyDCA$va#a{BU3(r!!_9Pgi-2i;E0Gnc>aY*pS>`Q?rAh~?1EzyTJYGW$G}lmao}<0 z*ztsMKbFF-eu*7V7=L>*IQr};=!64-(22VfRUDgdQhEn)OlmlIa$Ie2Y;;fQ`Hu@I zfnkcTBOIo7wu4S&m z25906w&OAX+4I*x=S)cx@nLS{Ea<$!lc4i^jf5`nN>y>}grsIYzzZ8p1Sgko23{od z2QT{69h&k4JD#xkT2pZ9acRd(%dr42+2&^C*nCS<--4GViMB9kqc6j-eDqIny6-OV zir&(Wx3a@U*jG79J6=Yuo#53LS(5%*49)y}SjDjmWZj(r&c2=rUXwczymn6lc-^{a z=z4KUC_ZeM>hZ>Z!_@D zH{YSUd(w`#>%txI?nCFHIol3F#Xaj^s5o|ky-BNK*q3+?tRJHR?+@DsJ}@u?da$ds zw*zyI)`Eoleq@ zch^M$zE`&%^nRr(&X)YRX+`E zR|OlMthOjjf}!@WY!$~YQ0MU=aNWBz!S=`6f$Qalf$MMehc;N*9old~TWF)H*zshI zN7n>946%SV=~WDE+Qq@hvH6=ddjNK9Xbu*emp8!RBzp~m)2Dk-=O@_lWG>f^fECAg zLS1vRpe?ptP;qR+mT5Dk7kv8dvVc z4&eVZt~~CUgFF4Fah3ZT{BPrGy_r$FqJca_TjVUP)m!!9;{~m$oxHC$++A*~wg1Wf z@x(+sxt`}!|Ep`2YwGiF*J|rM<9~Op`VO@)_MLU_|F!Ow_@DOG|3}>`_5*KeADUMC zPvfe^KaH!I`uX<%G_E`w$g7L)6lJbA3l8ajH?D}Oy$5B%g0g^-Qp=B&+J2;e-PaOJ z+1M&=UrVIEBHh7XHnvLJ*Ah8uOwt$lyT(>&`&uH$m8!ADzIdtnTI~8T9v<^QCEQvD__HUyGi9z7&luBb{H0z7`5&{_h)GV^s77 z{=Tmz;{O8hf8E$BZC^_)Wn-(feJ!z+jjhu5wM4#jjjhu5wb&nwm#VSFlCM;KEqeY- z)7YZyFHK(y&wnWzTSocywhpN13;cawOZr0({;wNbrR{5prEF}Kwy!0Y zvawa#zLv76VFI8WQo`0bfjV*foO4HYp{$T9y z8(Zg%^acLDuO%kq4*t5aRocFmxc(OI;BOmSrR{5prEF}Kwyz}$rE6@Jwy(whV7yd~ zEtY(x>T9v*uT+gK%Kp;yweb9xqOpbKEk$2Tdj9{uv1N-d@b`T!5r51ptp3}^){;;a z$Clu)`&wct8(XFAYl)?7Y?ZdJB?_f$Y?ZdJ#h(9CHMZE=m#VKt&wpteTa1*Nn zFGXVu*}%EfZPi|3hO-6tuq@ z{HL)+uTs@cX(<21f7M4dk_Yg7dF|q+awo0JAbA6AbqBc>8))JnA0_bR^=#mYg*c|C z8m~g_^~d&7a#ri9s8+o)<*tTpgUyN|MlIc>`saQefKH5V`92%(g1en zPWefKE~DKUkWltN2N;bkIn6k;;``|eW~YT|$2eab>k{O`L@ zS?Gx={B*>!C4jPJpo5lzl$L`IS`t!P7SedOG^En2Pua4F$)PZ4{BLMzAYrmtL^{Jw zuOXc|CYmvl-7FD=J`mo?c^M1ISjKL#%^ zmjqsZ`UE)rMHG02?h*51O{OzG;8iKdaeXdhQwtbYtJlLJb8b~|md_k;_IOt{eORN2 z0I%(D4PIx}8@yie1-$-8xP}ZHtqQ;!Z}$XmDmnn(e83UB<@Svt^4pqjDP{Gx19}*? z$G(Tdj`Vc!&VVamUEEagF3)##9=AI%2%J+dAH1hsC-7c=H+WzDp=$cD@6i(Qe!&C0 z|7Z;Oz+-#x!7XB{hAg?q1Q-s@p9nsjT?jrB=?~6JI0ZKJY70IZ`M8Lk#vN-`6MVea z3GfN?V(`gk8^9+&)gU+Sbosg9(^r3i&%BBRpA~oChvD3{IcoZFJ~a<~AtwTSF*+N3 zDXlm7@{q$CGUQKi0$=Hn3BEd@68KuJ1n~71AHdgtg@SL|Sr)O=xSMyof(w4V2aC6I z9bmY9_X-Sm)^#C2?rv@l_};9>;QK3=f**`12Yxtfx0*f}yLcdWOy!bPb_lJFKmP1MWG!WUT#YUzq;7Ah&gbtlhoih+iQZ~j`0P* zODYDxA2^QOxS}x+z#m%o2Y>8i0DrP+3I5!C4fyjH8}L_~JT;!buh+l9@a;=BGjQDZ zJ-5L>Zp;M#T#^g^wKoi09ABUz!|$ct!GD4dfJ-JdC1yW8wUU_UhjlF?6>7(WWgZS- z+3#SmNgeYddj5EmXVaJ;dwu6Ah?xcYlv zu|YC5lebr*1h zHn&9>8hon^L!;`s;6_F#4LLUcmH~D!R020Cko=owegHQ!tSllwNAViCc~(5wDR?i~ zSrZI)>7)fKRGwhhCVSXv9N$7IHh`gJxn(f4vNr*@dJzkDGq0~EN4GPB!EIj3!EHCW zgWH{X25!GNKtp~VHk|>xk8cm|ICl%!qrV-v)2Jk{Qt=1uDfZt~L??E(GQ!~HIv?y+ zbPU|J@)&T}+Ygx=$9MbC7u@~8QE-nd3UJT#wcuU{z18H`J1!C2XN3#cJ1`vFcS>b& zzs@r?`q@Sw+g!2#KC!GXuL z;K2*ZGdGSOvL+5ZH1Z8NXvPrmuwIvlMLu|RtePBE&2(@G={*=cynHG+)X^Or`f4IL z%%ToB>{^P38TgT}IdFJRGfq>>v%%57et;)hj{;A;dlx*ZI8jaK#pE6aPrerpj$N4vjyu#7 zJY`m{hWw_kas*Esk)gr!A3x0!hUr~IIEZ0Qi^x&a5H%j3&=oZvKSS8XXOFf8nI z4i3pD*fior@0k8IO1!oSst0up!I@aK957c;k_U|v? zwRJKvzV>-#4f(D6lK@_S>JxawgHZ6s4L87>j&ucYo-OKO*s?LCh)&!(5;Y#bZH^as zd$*|=-x1yjywhblbK`hjcQf#=N~rPp-Hl&@cfTKs@ja&J!Fw)yfcL&fiO28TX%E(4 zKBs0M`2FG{0fqxRx502Q7BwE9yC@laD1g(D!Bt; zW91m|qZX*~_(vZCz)z|O7=Lo5J-F}#m3Zv=e|iu#9{=phDL6b|(N@De_!ql2fnQER zjmN)Qff|p09axOA^P55VsR&jr7)|BIP9zNkHFJpO}#5|95_vk%5UKF-7V zr=QN!@Bg2Vp~mCCJn4-KeqA#Y4&RPDfxpiP1OHfC3H)<(f`%FRUo%kS@x{GE!M}%J z!}uRZ)OdVJ@ApN_Pv9(2;t9MXN<4voZBG7z%p#|Vopvn4Rqm-_u>M`Mhz6=2LX9Wb+$#oGTh$8V)ek)Y+fJ(nuCWR=o=`KaB|D81 zYE46pC)jnh0oNWn3;uN)egoI-nyn^3dpSxxp~r+J_oXXnhkkp5V6h8OGb}OTl#ZH}ki#T>-r$6&y@T}Vk?D=2}xN}K0u-EB% z%#9PeJV1>nblr#=Pv~~!4#vCB@y2+Mjo;Md*E1Y7p3rN~ZE)}Il`-BYJQwWU_!F@x z^zEJjLq99jc!H0EloLMhKWLbP;A<9wao@|=zypd<;|YE{_ksuJ2N#i_{~|4Tkj@hv z5W9z&IUz6wC7v)i041I1Q?F#AK}pnlI`7-l;dCj_@cjVAu^GpXJ7C*bJTc(x}E~8eu)}Sh~}$+qt7OC><}gxKZ7T34hK&0-0?*#v5jxtE_mMHRJuMV%%AG6 zCSA~eA~si+`TLc`vd<@g*Sv95(>!bUWN1j&U9beNUm}7xY<&;jC{7H4Vbg*uFl_ci zjVElGh#F7W+Nu%8xA|QzqKURwH3RQxg&I%T@f$UspsR`+Pte^rWp3;}UY{qyfA^sm z;GFxZn19bIlz774!xO>czG>UwpwIA!VSm^e4NY($9yOkDue91W!eCg{A4KoP&6}y7-Z|cEU-Z_A; z?n8+uTr2P@!t;N9*;F{(&^KZRPPiEp1TN5;fo}y*VP;Oa9fKN=HJMQ333vTb;|cfb zdSLv%+jjWhFR=$dtb0yP=RJIg5>GIeYy&?!Qi$=#50lk+{-12(G~`&Ar-sARIgi25 zGJV0%!%u);%#A7{zn9%n;|Z@u`G8+L%*FVd9xcFct=2O)PI%X(D)?OyN<5*+>?^n^ zKLX<)p5Kyw|Np4#4a28=)Of<@6esW(9cnz`tGW{STgm}7`yhM|bkvakP_G34Y-b7n z0#*6Bhuu=0(e zh&jqU_qJrh$vR(HQ>0zdLB5E7aH>d;%7<;$7>_x~&^u)V-W%Wsi8d$D(Xv3yU;;#d(W z{klT;&u4ibZ$B>3vp0TWZ~XG_<0<{>N8gS8nI9cY((ETU#Mi0(@7JNe4}B&5$&K<^ zy!4%Ru9L@N9scipukQa3d@uTv+TL5_O|{gF({Jx8pKNWK!gHKkkv892URCQ6C~x^+ zh8FA}d90w{8z}dzZxwLJ@=<(%cHA_1ZN7hGB3|@v%AqvvE*WzFg=BS zWvgY*ahnW$#sB?hval7!DE-PjtNy#9-k#n4IQGkbEvPnooT+u1``qgIw)Ot~w$fVO zKS)a7nhyHr`Zj*@7kmT4|KEIl?aB3Wduh4HX31mg@z1mueE3Mc&oOyTfp4!JeS$62 zvyCC+c)2$33`88o1aUvuwv&aIslE-jmTw{mvXUKwZF zM1Rr3EZX}2>o?x@q*t@4-4`gw$8=UiH`-{J;NMU&(PW-slAEbw($mp~$u=((lTUgZ zV!xbI#I1KROnLB3H#M=cVcL;1y7(w(<@D8CbYfpCr7Z13P#yMjjAmE)>H6>XX1~i` zd&iZ%_O9%;cV(}=D|_u-*=z60UVB&e+PkvX-j%)fuI#mUWv{&}d+lA>YwyZldsp__ zyRz5bmA&?^?6r4gue~dK?OoYx@5)|#SN7Vw|Jh!9*ZhCvwRidwcQa9@*C@@Z%PObq z_>z(^AL_d8%V#ej>#bdEEw8Q3dTCli`@qYrd^6@Cl#~R=avc0vi;(%DmFvekxzbUd zeROiwy1q1RWJ!OYgS9w0%N5qU4B9ST%$kTkGj^zaer_=O4o^|}-kEFk>m04}I~Zy7 zukWYwUpq$?V4*e!%#2V4zV!SlZxRTUhKAroy> zA#W^=p##3DLeIZ9hPAw{3fp?c7;c@b3SW?89Q7$f6)}3LG4h(I8tu1RHRjMtZ@YoYZ~L8 z%2iW_78|FYc&3`>{=gW&?u=@>oxv#1+M-IUT*ZS0VWUCo($%bN>L{r z+v=aUZBg{eKr28>s{myyK}xFuWh+8Ts{&;!LrSXyWh+EVs{~~$MM|qhN-IW6t42Ca zWJ^aMX!%HK2}x-gNi`<-p$TJ88Hp2~=7DE=ZvxLenGH^Knh#D~KNCF1atwG*Vi=ju1=Nfo<%mr}zgT2Hex58gazLiI`a9HIgC11wsIPmIfQu1X^ z9|F!SNrq;J$ErAXWIvRWZ*BL%;I(;D@~wAp2d~eRl5e9~9q>j`O1@3cImRN#Z65aB z$gyKfz8!RH4=MS!?fe7Y?jR-KjzvcBPBSU_bmNYJbpmXhzlt#CLT^pTP;cYjatA%&EDhnG8ok61~`mltaZ z&if%H-_ek!MvfgvZ%fH{+|vSl{D72vC+**YPcD~|?=*MWNZ031$4bd}#<&X(XG031 z=gwYIv0?7Kr<8mbHtWC_>q*IXX*uJ-f?Up*=%-}R<#!PmEp zhTg2=CmK0++?+2ZU%^K=@U2KG`EFme1>fl8SVJ4ng*W^IU(W5?TCQu4i<=>>lG%UwnGqR>X*qPtS^ee^N|f6V1n96LUX^QB?bwn0cfzQ)O^;F=wge0;6-L14SuNIt%HVkdCz-$*{b zZdg5V-DiGG+0Td6>+7N7*ahmHLh|tqobAC4HX!-WM)L7(9`**e9faiL+vPcd+qXgT@f|WNf!(W{896pz z$LSxy9l!qbwyjRXuEEgBh~(ovyT6B{XC9J|_j0%b_R2)^@mEpN~&_u+JGO`Fs_2 zVBbwr^7&Qz1NK|&3LW^PrjcV8@E|pSNuV z#jb%Ns7Ok_;E+5O$0iKEEhS%wXEr$GfRucp^=5)Y)1~AKV)eAFEr9w;T> zsH0865uJxZBe!%la%}$5Vtpz3#>{tsVXPn}UsR+yIO?&KeCnRB!Rn(@@@ks2@=d&4G$T)Ub=!q~uHJ;0sR3k&}E!=lDv=H}~Kj zVv(QcZi#{UYrm^glm69*TD+8R8DKKHs6w2Qt~Z*6AxZCLP|dEg`!C`Z2DfzZ!Y=V8Ojg)-b;^%|6eVGZr9YJHjJ8nqHr|Z}ktlKLk-|pH9@b0Bj z^5y)l0^T!SO1{0%TvQ^*F1&A;0|xY`DyxY1dq~N5VErfX!P-*tvzHkITRY}SB?2IM&xl&5L z7aQM$UpAGJ@6}=pBgZCsZ7wC>8}(c8n-@~@z4N&Yes@Mnz9PjgaM31~eC#~&qg5IV zALmNR_o-+S`15!v`M%svh2z)$Qu2K}FcJLSRZ6}e>HgrKm8InS71tL0>w}bhzXNN6 zf8X(jmUMPAa_qt-2VICofv;~%27zCJkdHrk9^7HZAtY9G^`Gg9QnJSLWU*R#5PpH@{0bKD|9P?*CA7<4o6b7p`NIs!T z`L5t9Gmw0O^`{W#$bM*XG?GuSx#k6~)*HzuRL^Mywsl1E2{qEpz%?z9d_t}0m*85j zkbFXI-}B(w*Zvsk_kW>ohZic2GzZBi)T?z4TrUmDC)EG74cuS?l22%OHyPY$0FqB= zoT~;qv_$d=O;-AXo7y4ygl4m%j2xT4Suv7NXgz-=2M`Gj^!2f*#+NIs#% zn3doTh5x*5t7D&d7&>0mK|8fhH*)L(owg(S1W%i(V9$j}KB4oMAh6e1B%jdbMkjDr zZzP}4ZErnrcPAvD&||fSievNjtcK(hdQG+z*8H*{vJ~D4cagl98gC}zQEa$;6U+D zsET7J4jI`8h9M86hZ}MhMO5VHs6R# zAHZSVq~sg9;~F?zVG13keQ)HLpBPa|O1{X+S73;IFD2iYK{?HHB}RZMjTtkIM*?+vAqBE+RF$sDw%v(nX~Z(#20j z2_+#^IDNje`J~ec7Mz#pe6s{ja#f@MSJqj{VF>gfH89COSK98`hkW0WuTk=bTxB&h43j z&f5Zfiu_uY6$bZx3EzV(O>}`X;d>a~j4m8X_=*l{NK719)YF4`$;x-=l3K!7I;#j> znnn1^MOOdQnmo$$3TJCAOCPWV1f3r2suMfloGm!jLw z`AV$s^WXm41B1`o2wz9N8Tw0r57y2?cZrD$zIqbAZ*ivR?`DLrD}j_`?><8%XdxWpoQI?rFuQ#q*%*E&bk6u0oe?owdv-a4&=KTVJk1HM@Li0Jm$2F4H zpfzoPkM}<@53Mz1H&*TKff5tnul*VLxXzNfXq^h+~13o@9_Y>yBT!4=ck9v$YFakbqxHkoD_yhR(2uY0|@e=sB z$@Hygle}xPxL^FW;+WGC)@b14W&`k=rDfRB%gH%41q z03RQHxCd>c%6n*sBf!TUJEG9z z*8(4(@G4tk;y@=K;N#ABBheF`fR9hQxDP#9ANcr`9V^gNz5pM0_4h)%K5+v+KGn?z zgQ=PRSUs#}%1m6~5$=xmNH4&6v9!aLY+ul!ZwfPP?DrU5(=jzWjeWJ?7Hc=ZBt|(F4=@5 zE^V`GJh;XTaox;{xMqI)^}P2ptl#(h{p$}!>oqg9`&mOP$>%HIm{d{-owbsin z|8m27r|L`XtFwEGzN;RrVh4x=dF#4jOJ3z97MAd8V{wfp>_Fqa8W{roF0hMjUQi1?$KX1L^|i9BJe8Ij|p ziQ4c?6Fu>@CT7lgP3*wmG$Tjt)EFPtJZnsI^D~zEcq%Kny1ZzitiUK2twKsGk3k_;Pd;<#PZQ{RPG#YbVeUJCS|lXc9^$3a&RAQ+;`RFQLVkT@oq2W(e-*s z37ox<@Ts{p=Kc#|Y^j|z?#M+U@xXH_Y3(jy{E7?Ggjq(R-^8hsq#w6aPL4aTP3gBp zp45G(Hr0EQJlS)JHqCB`JcXa6o$|Duoc{X|ZN`~8^3-GPw9|?Lgz4Kuq!~sxVPWH%{UPD-|ulJ!u8>K>(>50t$nn<;;YtdlOn(MU07CcOy2O>w1npaS)*plU*D%GMO7Os zSyP^xugfVxZ}F)u?`w3*s#Y{%i-)zdq5NgxLsIEsDS?!dSf1l4 zi|&6QyXZ}E-L`C~(XpZ>+tL_V5o#Ms6W2ek9co-&5u|QOX}ek$y{ItT*lYe=_Ph_R zQ4(YH{-6=PKdD@eWjAYyPAzz=8pZ}b8k+mBq$2MRp0f5Yw89k<(ot5z14*d zFqlI>UDt)Rj5UWntTKnY%+!S+ziJ-#Hbxh*!DJqOwVN(-&T{jJy^VEIqo$dof3nua z1TE9W&h#*kY&BIE7o{?fa){K$cPcl>zv!$>sDIj=a9*Pu^JSZP><&&h?oPHjF|UC+ z>Eu@3_%y~mV8W(Fx`{&`nDukV>yq0ZH>ZpqsGH=n$(-8DPnY`jm@dsX$eea}qb}Xi z*PMQMo-U)p**taiSlzS>Rp#k4`|D;DUNz5*YNazQHkp+lFx|w9f!21WQPbGZJnfBq zrTvl89!Y7Rq_kJ4vR_i#GgR3(DeWDq?4Oi8ASEA2$qQ2QgOofWC0|GlWDb2n29c6U zq+}E+nMF#5k&|~r2`?QLm{PuVVK&==TJEv3eX9W(kY3~chvPn zr?zZ!F<=(ELriw&8{)TtsrvWq^YiQ+(JzA2nfmLvWgI(QOb+M%`d-exSxtO7kh2xF zA)K0>V`?A5wYOrYo0Npl!E6(8!e(xjczFxgMD*Iqg|kgf-*4sGvn(q*zyh=OfSu=U;K8-Z)tgW1w$4N=sp2_1UKa?gcwGsME9CA|9PkJgOx7{M8 zgq#y5xh#}Y+Y}0^UpH&hYUT)OcNb`%hZD6Kw-?J(R|jdQ{i2tr&-B&KST$Il znb2P{O!t>fpYC$iS@uVhT^To-r)`nfv^7%N9w}{+l(tDqTZJmyC8aHs(zZ!y>!h@O zQnG-QY#=2oNXZUTvV_z?zR(xs4Jr9UN*7E zl9#09CnhcvTyEAXty+Tib-If7?*d=H*u{R*7b#6?ijnXK|F;CZTd;CUza zg6BWH30|=CvX=4|9zO_P^y40I_J(|c)@Od0n1{gPABtrKnI8wHgLB5O1TXQ80OtnH z02|%XB+6UbDjK}>eK%bBTfrC4*fPYEO1t0ox zFZggop+tEl6GgDxAqRY9U1}SMa%I4EX%W58w;4++{|&E_N#gn`6F$FL@jSUlz*1SL|;} zl=qwFDEPN$2f^2v4d82M^TCGePmTym9&_Vo0|GaHT@SvsC>C70H4A(@u{-$Af&}p0 z&={>Ujk!0rFSyLV8~A?T7T^c98iRjt-CI`DnBU)6ga4@E1^(m8V`2kS{@NaaM~76B z5?~%*dI5g2svP`u*G2HN>8HWZm+S(+h};H#IrW0TC|5;bHn=iE0>5&f0Dj#e7hGkv zNK1Kd>ZXF<{4ri(<@^8b$1nu`JTVXn?{2jRzu)Kw{%|Z*W|Zq=R!#7y^#S0|quzqQ z%yt8R4R)3&@7u_a#4OXQikM})-3D_GSHZl;VKD!~B+>e;m3^r|iB{*ABf2D?XX0@t5@ z2Hc?cF)igajNA^^G}{Pn6u21dp`K?zz_ZCVnG!u8jRiNc$_6(%)gRpSj|t#rn_GdK zpU{E5a{5X%&wFEQa0`74aLcS(;8r>Zu+IcPf%1HPE5O>|nqWUi!zBb-w|a{}>-R6U zl<4np9qfO@3~u|P7#y&FH@MyTmEiWvmV&?AXObCZUWfEFaL2sm;K1;3aHq6s;LcrB zCCcj(HUcad8igUyHJ}Ru-Prcv?hPXaO6-2WF1W|H&fuQ%XK=538gTDzb+nY%r-TFd zUGNFqZ}mfP|FL(#17@BC2lXuh2PZs`8D*Yfpx0UibiI!wFvxZ`cyO~#;K8rgNR$|= zo(~SaJPSPJ(Kv9}?l|yJ^E`p_!gG6qhe>0>5h=~U!*lwBBL~}pN9bD-8`!9hO|{Aw zY_!f12{H9-z_Grsz_A~ng5&Bq$dng%>l}F0`wDRUfkN=;8<)Td>(5A(H)ekkcEV3dgyJqLsJUDLqHygxXlQ8+l|`5-OjrLrBs zspr~&C*OAmr)_ruPdU?Gro8k;DDl{gZ7A{Bsfjl+K5fBg8K3{@p>HKhoH6z;Cd~A| z4mR{fjmKuzDhAK;T8HsjZ&re5*FcTO&i-u%c+Ts!0^KKTPc(S$<=NnQ`6%(&`MXi# zu?uEIX(?}EZf^mf|3xDN1hS{a$u#goClB!A;XT1W)>naZIyM96d@cv)*0TlYmU9x# zGk!#k$1Xkb5d71v^BB+DcoHlgdn!=gvMiK%?DF;J3<#`$pBF5KxhJx3=3<9sOrk5FI-uer^;0@0QgEtmBgEyV?2X7Wp;;~z{ z8)_r4b&8`z6SpmT5B@pq6?pq(7lGnC0#M_zg`ueN*qycaVqEe^jmJuVis0R~4kCZ| zJ=Az?(VKkmo)VOJ?B3u0+VH+L=~}w4$*?CD3HuGG@z~;g)OhTH=s+YKoDnTk;xAnq zfDes8jmI8t-T@1ibn?WwY~v0-Qr`-E49aHjJOutp}fr&jz2@qQ+w{1f#}dFFGY)-0ZVZW|RfZ zm3_gNt0#dk-;9?ie&uB___yNT#0K{21uX*CmU|)Lx+zFd2G|=JsPWjFB5FMLR`?5y zmrilU`0e%=!FR%{z;_#=#$)dVTm_f07qoPr`?XEr`}ZaAgKt-4ivNCuN<8KB|M1>k z1pZi?iiGkK5&URjjzo!%*JOa7j6;pbJ~a#nKkFBY@#oQ_W{~h^_XjOae0%Bv_|M!@@H^?N%qa2qDTlxx za!}*39|x}if6^bt_~(uXCCdAv+W`JrFCY9(I~V-zlYy9HoyHJzY-uch|8v}jEP(?2 zLDYDh>ShAiYGo5lurBTkR?q3Cr97MEsPQ=4QK<1ay9@`6+xM`>c(svUGUYjVqQ>K@ zcSDKCIr3*P?$qc7#+`oOZa|<0iyDusaSk;eSM&ZZB-GlzO`;pQ95I4xXQRgBT(?OA z#odxn<8gHs=7Q@E&>_FxxKyxv+wr)6{eEHK1}+1^4ZTs~aSh)Z{19klR~HG5uGR#5 zR0YV4vViBFx8TNC+`vtiUI#bbjT(<@Hls?Syym&6@i?y$SHa#Xhmqf+lL_2%c&R{n zt?Hx1<9s?E0{a-gOhbaU-Wmk7<;%2`==W(Bxb;cYcwC!O)Oei#rbvvpJ=hcD0dqTp z+ii%IXSV7%wz`QTpLw}X36L5;`t$zBZZJ9Hi%pkI<6+`rve z@PHvG@eEv$YkvfS+fI-fW#Qm=t-v}L9avY^SEBf!x2?g0{D zCHmDxAZZ|KJZ`+xXYd4HjX*b=PAWaZ}%!%h7z9tqQi&PT#@57cTe2&~oV3hl3&3Xo&TYwUeo0o_ZkDG6JgbS`@yadytw%#nc|C|=74i-ioiK%Qou_dqsHTMi%{cn z#?wj-hJo`Yp~U0FC5AMC29|~RBVlLa-BfBk<^Jne9>n;D;=ACD)AGTa49ijDahpdK zOO&`JV;vH<_E-ttHf#>~=f|JM%z`pS+1Wb!i<8k{3l!J>CZDmGT@IcGc;Dh~8<8i;Z zpvL14d85YT4!u1oQC^ANPUM$d-2#@Y&IuGhvKJ*Dck~KMJnq;}Ni=TYju+)1;l$)% zOgL#w0iTLW)Y8P$lS0A2b`Aoc2?+q7ZQu(&7dTj^ymMcj!57^9!58jSfiHfp4K|;0 zlql~~DM~!<@+K96lmg&ULyGBsDt2JO*6oy zJ@Ub~9Z}0%|<& zLsSyRKWbVa|5N8+@Ml}pc-)r;DDk*2PrUE|U%%GC__tr}QIYV>Bh+|2y9G5K&z(Yz z$Mf?q(ga?$sY)QVN`6iY@YZut$Mzg@0<&+=8F=Kul6bx zT-{NW=|0sj8Nv{7tjIvX=|GA^1I`zRgKMk^1=rl!30!Mx0N7<&gg|+)okNMo*LQsYu5Wlyii8I5jw8_U=pnGCtW0K`~(JExl0U z@rIVqP~-7Fwy5!VpR;OA@O{z(6SR98V?n=P9l))ZqQ>LfY^ebIPkM~;wz>8)qbv{* zat7Qk88sf?zRh;`w29dEiU^6{LdEoEs-S=v&Twv?qUWob)U+ESLbl%*|YX-iq!QkJ%qr7dM? zOIg}dmbR3oEoEs-S=v&Twv?qUWob)U+ESLbl%*~8?`Qm%9Dm@ar8_Yic~1mKO6Xcu{kTvoQr6;+7N-%)yb8r5E|o-$Rr8$;7c=@z7P8`0NOyO3WXpYLz!e)kX>Vk5P@`Sd~c${cdk)v9mJ7 z|6pclv#$CfwemBEe0f|M_T_-`y4F;E_?=bEu#=DU5r<|n!}AwcM&`#TF9U1zQ8T(J z@06daj2Y3GiS6WHIkK0K8Q1=#epFMB%y`G~m7}dynF;Li%7j14GsoOW(vLlHI&<94 znw5zgw`C@6+o&JE_!KiCy@r0`gsqHDA6#6S9K1+*XSct8lFxW1)pfsqa`l1CXL@-JNlkC6DMLpGe6oy2vk5@{BI>jg-8j%RmOw7i1zS8A(cJl9HjMWGX2c3suY| zC4))HWKuGkl*}e2!%4|>y2yA^GGDn23>^S{MTbBa9Rw*I23>R@q;x1yJB>7YpIut@2^Na@f>>EKA|@bKxO6BL^+RsE=9m_|0Ht)eQ)R#d)+Yni+^sixGl zUaI^+3V1Gdt*iV%%KI18GT9tZC3-kTo_+nbN|xi5b7(BizO5R=MvBjGtLn3j zO?G!w*KOGX@ji^e!+~R7%+GgHcA1l*a zA8X}RZ>?BR&AN$6Zm2%1Vs&EX8!LCQVGV0L(P6oDb(7p!t?Dlpx~go%a}(9|)yng8 zyvjirJ-+W z=vx~4mWIBip>JvETN?V7hQ6htZ)xaT8v2%o{{OXxey9J=d!~YTe;}(db>66s;!P{I zsO#_!iwf9p-zLRS=k0izDF3YP;jM&}zvmmBPJiZC8ByAA$IBQM{k`tc{-KnpTz1Nr zt^CAYyR7Oce<9VuUI{5(HpXE^&E0`w%yxBi$A7k7S&m^g_Ym`TsJqu-{+UQUS0q2% fq1OFp$&GlsUj4-6dun%4I;8v)fFozrkq!PI$4(X% diff --git a/libc/zoneinfo/zoneinfo.idx b/libc/zoneinfo/zoneinfo.idx index 0483c285f4d1bea496805a5aff9ecadc7d7dc931..b70213292fede0f7c58c3ae8a8bd8fd4241acb19 100644 GIT binary patch delta 4441 zcmYM%e_WJRz6bD!Gs7^#fT*Y=q7FE$U@*dwl(kVXQCYMl&PhS*6Ahg zwT(95gGF9y@kYD2;EKi8RMfJ?wqz7|Wmharx>1WtExWNUwPg2mp1H5Rf9RX<^PK0` zch32qb3E7m8gKZun+(Eag4y5@*54!9AcgHMl#s|eG?A!G43NdKFdNA`a1#4&l#%FY zG#jO9$(x#{UB{5oA!1r!@s}cYJyOZiRdAEIOK2hS0~jI+hY)I#A~6JMij$}yawXan z0~jXDa}gFGMbZqUE1pFqxqTJd72jfntT+j)S&Ef=kwKD8@F;rFLB9VBjFMH2h?pkD z9i?y)rwdhNwGCcHC&tK{Bd`VT4NQsCH2)RaiZ4B=30x}D?nh^!6dOY@PPRnB9wfz% z?Z_s%kD;36MWZW7n)3dlX{H|_&oT!g7OSZ+LDP)U4Ci8HI$IwwxWr=+>HV9>MEFLNby=EoTM-TW#sLBXeRF+#2~qt0?SM( z2CpHN@O$0l^1EmuA0EXJ`6vmYv!u8dgfzm}svyim8~O4ohRJw1!otX9q?5^?qmq2> zLOYqVV1yVK!8%(Ca||-bjOXDY;j7W1xQtO^k4MBDDZY0GF5;X-6*&rc$&zO=DGRFKG7Xd_JbFj@W~!Xh~wGm%c#o5>SkWJcOLpAY! zg)Y*SiwW{}EMmSV#XH|3hw!=@!p_@G`rpSSx%?I4mPpgkPEE51pWrMg&F&8CdV1&GS2-aoN^hS`TMV^FfnL~X10zAv4_%sO}PkG7}RGM#C-~XBgl{CwbvJj1%)R*rgOfr;+WWngl!0 zMdm(^2@>)cVwUSs4zd0dH6+}EZpCR#E|;c=Tbj0bEfSI(A~gkhNz$}d9odBj9@%p* zdXm^-H(`qGyMTn-*>7i|;Pym=X?+|T7#@BZy~G_-w?dlSS9z^Jl2jKh80r?yF-dU??yVgl!Zz% zXhl02O2Y`bvJqCNREf(VS4Q9=|5J+&^4SWElFwg6#A+!9$w=_fpW7K4N6x$dUd^Z>lkh;M$$Sd zYT8hw!W|3HsJ!nL^egi+VNBCcCzIkCC?SU$&_oV)`T_skyZm&QoOzhT|OohClIqmiq2WcQJg>xd1EfR70+Ojyg3(f zcaamwBfp!4dPOaINY`{sDLA2T<qKw*IP_gfD)aJXC~EQZqc2_VsJo445y3l_)pt-C-M(P zs?lI#QU-JN+*tDTu92E217BanMh~i%DBBucYcm@B*=bsQ4TgM#=JVEDmLM%(pWrGO zszTMKAASzQ%DWy#*!^^?6Y07=s8qUkqFr|aBg$>Hu(!M%H?70T*DOVXQy~ z{r~1|3u-sVgal+wBx!0KXRzQ^@?ZBBxNAN>6e6Sc&>Vhq|_# zCwT27Ip#!Mku>?L6$*;D@w$fkA{MDv(NiR6w;tp7wTkl*9#{VE;=RSYmu)EChf~WJ z1_ZQm7rAgRl8PPT{8K0>=G40w4T@3plHUKQ+b6~Q(MVE^p@@9&IT}gdF7%WBSQz)~ z2}&jxyeJ|6y&FyBqCW=6zzmp6*#G?CRJ@Ba^5-LHCYM?Ld$8#38-7xd^x8RfPLnFX2*ujBu}MJFIV^G70w<#|bmk zNw`58C)_&P%k{;0HsQ{w+DBKp48GLL-WK)k^Ie{-OyBR_zSE@2XpCF0ra6T5qsA=* ZKjADCuuUIMp6zNl1^K_r!sr`e{|ih4-+KT6 delta 4303 zcmYM%e_WJB9tZG;*=0pslnQl4P?S~C5O+t^RZ&q;P*7I|MMYf|6$npMG)^hW@JAOZ zw;#MHv8XsB;);wj@+hfD(Xj9pC6`p3N6IM~nHH9>y3ag&ukH_c@AsK`=6RlZX1+5| z=?$;^>t59+gYX^+A47x~_%ApNQVhL zBfru#?R~TvBgCjOn7yS4n}%pIS|W!$_z5aVcoW*lxD5E4q_CL~t8gQi*vFwt(S&v) zmm#2+6ytj%UU39@BvPVUaR(h_f*TeeDJHH)qT*NNlS%ccA&#ffNhTK~sJ9eTHo-}z z#-o5lSx~E}K^K|!FhYDc`cAiMnpmc7{!&Ac?-&ui8+E=?#Ee2WnL8EMK2j{sLJC>= zA___Bbkz5erd51!!&11+1JKuOF}ap%nsEfffrv2=6luFrWDY{Ed4T7q6zQjUBoJSj zEh6i#{_}o6i`eQwxStd|_Q6H|*sd46=4UZII+X2&y88y=s$YQEI}7(%zW*?6eWm!* zZPpINqkS#n@IsXE2o9l%6g%Dhq$rIv@2hd7BNrCx(DLxa3CVbu; z6TtCRxV*bRFUgFXeV9c5fDIbA)efR8F}PR2C7NV2y_t5 z0ZV@=j5Co){OXWT1}sI5;ubnd$Q%R>;0QefCz+Ux0&+=*dQq$H6oSp3PogpFd7C) z)8Slp^eeCh2EacsQ2b>W!ULr^mJe5;G?n{ln)eR9uqrUn^vYh=eM%48S+}|$?y)Z0 zvkhjicO#9wn~4%~W(t}})?Igy6pe!sNiOo$lFPrMl<@k^gq>#$W^V_>q4*J5 zBRrvnu!_$RDQ*=YirmRWHu zPryG!ih#k0B?CH~R^8|uIgW(<8U+npoKc#pg3{K)+j{;IM3AKt& zbdjIb5O^Evo+`@$FIOTLW);HkU}cGQAkd3Vw1Nk zahufTAZ(-*_3=n0JgrdflLHfpRv+&zurQFep)wv;Xu5i6?tUd zG*pulx#%Eo_J?JhG@X21(}p?V92X%jT|@pjDcW|ThO~c-PSVkaAe)YpT-%HSa$_87 z6}QkuemIB_J4f#nB-!l-!@cKmkfC-Gb!7HlbQ7yv4DNREL@yJA!@Q8IG8Y86`F&+2F z=nr9=z}L|WX%p-Q@2|f?2}64&n#g$Jo+wS@Kh-p!BXCTN5K-w!pU8>OAEjhwF`7xt zAQ&e}5nBhxBzkC0-WT)c@&T*Qc+6^ zZRjG0K0?TJE?EPSH2wEJDx4l6p4p5#7Lac8tPR!~`s1XKqq|Y)p?-$+eeNAi^V;Y2 zR*nW+^k{>zY!*_Z?FQoub`&Z9`Y{@mm8;OBe0?UuXVMxQT*~)OqF8w@4~@#^Yq+Pp zcnP*y9@A#&hu*~JS{RW97YB2H% z7rh$71ky>E5QFCF8OKSu@)i&#v0B2!(?wV%BtcI+NeRE_#lnOLQ}A<|7CH{~2@z@u z?M{$>v+vWiUP)@-HG4@C7P&(H9AYwfCvHGWB8T~96eeykc)xoYw;3j1MA&?tRI)S< zMT!s6K$gX!M{y3}3wY%OxD@A5OqR!?QSl+}krgqpEhO(DjjWu362&`cBFPusPVVfc zB2sY*>15S3lq%}cOjgIhxQLvAgRGg0EJY*Ah${{)iZk$8%tm9m)1y-Y>nSX}!7sT{_m_{kI{=U&^wR`%t4~zji8{DiE|xZ`QetVe8{4 z@PxH0WJ>Gugdr&mukS&UvO5_El|8Fbm!jWHH{q>Wmvc`Lh!irk5`|>+N2p)Us%-c+ z!|^{NYy~&xBaupCkD!P!H8l{f9X(|6O@y!18+8#rS~1x)0gYtK=eS39N!U{LD2=c$ zN)$B9V>M@ zk*EC6cvS1&M2GVFE?Cyn8@-XJdmQ=7t{Bwlwxd(|ts6mU^yXu5rrG^W-wokS9n&tK z(&lc1ecLcK7mZBNwQ0X6=;G5aQX-CUQC>W5g@$IvU`WgOff`~3f7 zYCeX|4KtX&`vzSvk561_03BmL$$lC-zd!kxal*+{(3S)Td=~D3VIoO z^{GVICMo)dBb9ImRkTU=+dqMqKBBtCxg^Qqgni=(;l(7sNt%3@uxJ@<=@Ejby3+Mi zi%D=_G^We{r66}kgowTXTZR;~jv*~W`o?7N*erZ*}LFNrbmf{A=2)m?(B<_Mw78B$ML@CZ9n=JSg zOEIEt{k~A1?iYEB~K_`|Zry^JJ8>&c3DB20n3)sTx zxZ59Vj{op4~+P&J=Bd6 Date: Thu, 12 Sep 2013 21:47:20 -0700 Subject: [PATCH 67/77] Use kernel default for initial thread size Bug: 10697851 Change-Id: I8d980f5e0b584799536f6e6b891056c968d26cdf --- libc/bionic/libc_init_common.cpp | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/libc/bionic/libc_init_common.cpp b/libc/bionic/libc_init_common.cpp index 1fc490e23..c10abadf7 100644 --- a/libc/bionic/libc_init_common.cpp +++ b/libc/bionic/libc_init_common.cpp @@ -37,6 +37,8 @@ #include #include #include +#include +#include #include #include "atexit.h" @@ -62,6 +64,21 @@ uintptr_t __stack_chk_guard = 0; unsigned int __page_size = PAGE_SIZE; unsigned int __page_shift = PAGE_SHIFT; +static size_t get_stack_size() { + const size_t minimal_stack_size = 128 * 1024; + size_t stack_size = minimal_stack_size; + struct rlimit stack_limit; + int rlimit_result = getrlimit(RLIMIT_STACK, &stack_limit); + if ((rlimit_result == 0) && (stack_limit.rlim_cur != RLIM_INFINITY)) { + stack_size = stack_limit.rlim_cur; + stack_size = (stack_size & ~(PAGE_SIZE - 1)); + if (stack_size < minimal_stack_size) { + stack_size = minimal_stack_size; + } + } + return stack_size; +} + /* Init TLS for the initial thread. Called by the linker _before_ libc is mapped * in memory. Beware: all writes to libc globals from this function will * apply to linker-private copies and will not be visible from libc later on. @@ -76,9 +93,9 @@ unsigned int __page_shift = PAGE_SHIFT; void __libc_init_tls(KernelArgumentBlock& args) { __libc_auxv = args.auxv; - unsigned stack_top = (__get_sp() & ~(PAGE_SIZE - 1)) + PAGE_SIZE; - unsigned stack_size = 128 * 1024; - unsigned stack_bottom = stack_top - stack_size; + uintptr_t stack_top = (__get_sp() & ~(PAGE_SIZE - 1)) + PAGE_SIZE; + size_t stack_size = get_stack_size(); + uintptr_t stack_bottom = stack_top - stack_size; static void* tls[BIONIC_TLS_SLOTS]; static pthread_internal_t thread; From 6d1f85dcdcf32117332b48563357d311a2886c30 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Thu, 15 Aug 2013 12:54:40 -0700 Subject: [PATCH 68/77] Change hard-coded temporary directory. The properties tests creates a temporary directory in /data/nativetest, but this directory might not exist in all circumstances. Change this to create the temporary directory in /data/local/tmp. (cherry picked from commit bd6dc6a8864a1997eb8608030ac816740169c463) Change-Id: I15144ece7ffb5c29eded9a1ef399db630f6af5ab --- tests/property_benchmark.cpp | 4 ++-- tests/system_properties_test.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/property_benchmark.cpp b/tests/property_benchmark.cpp index d10be915e..4311a1db4 100644 --- a/tests/property_benchmark.cpp +++ b/tests/property_benchmark.cpp @@ -32,10 +32,10 @@ struct LocalPropertyTestState { LocalPropertyTestState(int nprops) : nprops(nprops), valid(false) { static const char prop_name_chars[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ.-_"; - char dir_template[] = "/data/nativetest/prop-XXXXXX"; + char dir_template[] = "/data/local/tmp/prop-XXXXXX"; char *dirname = mkdtemp(dir_template); if (!dirname) { - perror("making temp file for test state failed (is /data/nativetest writable?)"); + perror("making temp file for test state failed (is /data/local/tmp writable?)"); return; } diff --git a/tests/system_properties_test.cpp b/tests/system_properties_test.cpp index b9256c664..0a8cfa519 100644 --- a/tests/system_properties_test.cpp +++ b/tests/system_properties_test.cpp @@ -28,10 +28,10 @@ extern void *__system_property_area__; struct LocalPropertyTestState { LocalPropertyTestState() : valid(false) { - char dir_template[] = "/data/nativetest/prop-XXXXXX"; + char dir_template[] = "/data/local/tmp/prop-XXXXXX"; char *dirname = mkdtemp(dir_template); if (!dirname) { - perror("making temp file for test state failed (is /data/nativetest writable?)"); + perror("making temp file for test state failed (is /data/local/tmp writable?)"); return; } From 7bfef355b1590cb8ac68d8caa97eaf0018191db8 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Tue, 27 Aug 2013 17:21:37 -0700 Subject: [PATCH 69/77] Change hard-coded directory. The tests are using /data/data which is not accessible to a non-root user. Change this to /data/local/tmp which is accessible to all users. Bug: 8291716 (cherry picked from commit 5227bb363dfdd0a4570cfa3d0735744daf7d83c2) Change-Id: I83bf70aa8edd21b00321363d7ddcb65a5f048ba5 --- tests/statvfs_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/statvfs_test.cpp b/tests/statvfs_test.cpp index 8afc6fd20..31ce66aee 100644 --- a/tests/statvfs_test.cpp +++ b/tests/statvfs_test.cpp @@ -35,7 +35,7 @@ TEST(statvfs, statvfs) { #endif #if __BIONIC__ - ASSERT_EQ(0, statvfs("/data/data", &sb)); + ASSERT_EQ(0, statvfs("/data/local/tmp", &sb)); ASSERT_NE(0U, sb.f_bfree); ASSERT_NE(0U, sb.f_ffree); ASSERT_NE(0U, sb.f_fsid); @@ -59,7 +59,7 @@ TEST(statvfs, fstatvfs) { #endif #if __BIONIC__ - fd = open("/data/data", O_RDONLY); + fd = open("/data/local/tmp", O_RDONLY); ASSERT_EQ(0, fstatvfs(fd, &sb)); close(fd); ASSERT_NE(0U, sb.f_bfree); From f4af911065b1b392085b857b891e5496bc75f478 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Tue, 27 Aug 2013 14:32:15 -0700 Subject: [PATCH 70/77] Create bionic unit test library for use with CTS. Bug: 8291716 (cherry picked from commit 153d92765bb693541abf49a2a1d31335e2c90348) Change-Id: Ia53b85b95621b7000034e4c6259d6586d0987dee --- tests/Android.mk | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/Android.mk b/tests/Android.mk index 902bf6959..400d97923 100644 --- a/tests/Android.mk +++ b/tests/Android.mk @@ -122,13 +122,26 @@ include $(BUILD_NATIVE_TEST) include $(CLEAR_VARS) LOCAL_MODULE := bionic-unit-tests-static LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk -LOCAL_CFLAGS += $(test_c_flags) LOCAL_FORCE_STATIC_EXECUTABLE := true -LOCAL_SRC_FILES := $(test_src_files) +LOCAL_WHOLE_STATIC_LIBRARIES += libBionicTests LOCAL_STATIC_LIBRARIES += libstlport_static libstdc++ libm libc bionic-unit-tests-unwind-test-impl -LOCAL_WHOLE_STATIC_LIBRARIES := $(test_fortify_static_libraries) include $(BUILD_NATIVE_TEST) +# ----------------------------------------------------------------------------- +# We build the static unit tests as a library so they can be used both for +# bionic-unit-tests-static and also as part of CTS. +# ----------------------------------------------------------------------------- + +include $(CLEAR_VARS) +LOCAL_MODULE := libBionicTests +LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk +LOCAL_CFLAGS += $(test_c_flags) +LOCAL_SRC_FILES := $(test_src_files) +LOCAL_CFLAGS += -DGTEST_OS_LINUX_ANDROID -DGTEST_HAS_STD_STRING +LOCAL_C_INCLUDES += bionic bionic/libstdc++/include external/gtest/include external/stlport/stlport +LOCAL_WHOLE_STATIC_LIBRARIES := $(test_fortify_static_libraries) +include $(BUILD_STATIC_LIBRARY) + # ----------------------------------------------------------------------------- # Test library for the unit tests. # ----------------------------------------------------------------------------- From 34c2a9fc37848d446bf91ce61783884232fffeb1 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Thu, 29 Aug 2013 11:37:33 -0700 Subject: [PATCH 71/77] Move stack unwinding test into library. Bug: 8291716 (cherry picked from commit 8240bed918722944c9985bc0cca89eb44c4f6c91) Change-Id: I1da2d0757ed304c352229f9818c052925d4e976a --- tests/Android.mk | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/Android.mk b/tests/Android.mk index 400d97923..534aa3017 100644 --- a/tests/Android.mk +++ b/tests/Android.mk @@ -124,7 +124,7 @@ LOCAL_MODULE := bionic-unit-tests-static LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk LOCAL_FORCE_STATIC_EXECUTABLE := true LOCAL_WHOLE_STATIC_LIBRARIES += libBionicTests -LOCAL_STATIC_LIBRARIES += libstlport_static libstdc++ libm libc bionic-unit-tests-unwind-test-impl +LOCAL_STATIC_LIBRARIES += libstlport_static libstdc++ libm libc include $(BUILD_NATIVE_TEST) # ----------------------------------------------------------------------------- @@ -137,9 +137,19 @@ LOCAL_MODULE := libBionicTests LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk LOCAL_CFLAGS += $(test_c_flags) LOCAL_SRC_FILES := $(test_src_files) -LOCAL_CFLAGS += -DGTEST_OS_LINUX_ANDROID -DGTEST_HAS_STD_STRING -LOCAL_C_INCLUDES += bionic bionic/libstdc++/include external/gtest/include external/stlport/stlport -LOCAL_WHOLE_STATIC_LIBRARIES := $(test_fortify_static_libraries) +LOCAL_CFLAGS += \ + -DGTEST_OS_LINUX_ANDROID \ + -DGTEST_HAS_STD_STRING \ + +LOCAL_C_INCLUDES += \ + bionic bionic/libstdc++/include \ + external/gtest/include \ + external/stlport/stlport \ + +LOCAL_WHOLE_STATIC_LIBRARIES := \ + $(test_fortify_static_libraries) \ + bionic-unit-tests-unwind-test-impl \ + include $(BUILD_STATIC_LIBRARY) # ----------------------------------------------------------------------------- From 427e8cf90248f7da0a789fb3b4fb9a859c19aacd Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Fri, 27 Sep 2013 14:18:36 -0700 Subject: [PATCH 72/77] Fix line that got removed by merge. (cherry picked from commit f14d71fcf74e755ece4f8ca2ca2b7483424aa432) Change-Id: Id3dd54d9efd87ef2bbb2c90adae2ce3911695027 --- tests/Android.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/Android.mk b/tests/Android.mk index 20262f42c..f1e67f87d 100644 --- a/tests/Android.mk +++ b/tests/Android.mk @@ -150,6 +150,7 @@ LOCAL_C_INCLUDES += \ LOCAL_WHOLE_STATIC_LIBRARIES := \ $(test_fortify_static_libraries) \ bionic-unit-tests-unwind-test-impl \ + include $(BUILD_STATIC_LIBRARY) # ----------------------------------------------------------------------------- From 21f59276058a897108bdaa72d3c4089259b3292f Mon Sep 17 00:00:00 2001 From: Elliott Hughes Date: Thu, 26 Sep 2013 22:41:09 -0700 Subject: [PATCH 73/77] Upgrade to tzdata2013e. From the release notes: Changes affecting near-future time stamps This year Fiji will start DST on October 27, not October 20. (Thanks to David Wheeler for the heads-up.) For now, guess that Fiji will continue to spring forward the Sunday before the fourth Monday in October. Changes affecting time stamps before 1970 Pacific/Johnston is now a link to Pacific/Honolulu. This corrects some errors before 1947. Some zones have been turned into links, when they differ from existing zones only in older data that was likely invented or that differs only in LMT or transition from LMT. These changes affect only time stamps before 1943. The affected zones are: Africa/Juba, America/Anguilla, America/Aruba, America/Dominica, America/Grenada, America/Guadeloupe, America/Marigot, America/Montserrat, America/St_Barthelemy, America/St_Kitts, America/St_Lucia, America/St_Thomas, America/St_Vincent, America/Tortola, and Europe/Vaduz. (Thanks to Alois Treindl for confirming that the old Europe/Vaduz zone was wrong and the new link is better for WWII-era times.) Change Kingston Mean Time from -5:07:12 to -5:07:11. This affects America/Cayman, America/Jamaica and America/Grand_Turk time stamps from 1890 to 1912. Change the UT offset of Bern Mean Time from 0:29:44 to 0:29:46. This affects Europe/Zurich time stamps from 1853 to 1894. (Thanks to Alois Treindl). Change the date of the circa-1850 Zurich transition from 1849-09-12 to 1853-07-16, overriding Shanks with data from Messerli about postal and telegraph time in Switzerland. Data changes affecting behavior of tzselect and similar programs Country code BQ is now called the more-common name "Caribbean Netherlands" rather than the more-official "Bonaire, St Eustatius & Saba". Remove from zone.tab the names America/Montreal, America/Shiprock, and Antarctica/South_Pole, as they are equivalent to existing same-country-code zones for post-1970 time stamps. The data for these names are unchanged, so the names continue to work as before. (cherry picked from commit 35b123ef1e2ffe48275cb24708f9d88709486838) Change-Id: Ieb2627cc817db93280ceabe4034800bf36ca3f5f --- libc/zoneinfo/tzdata | Bin 567299 -> 560770 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/libc/zoneinfo/tzdata b/libc/zoneinfo/tzdata index bce6e500290ad5675996c1311360650fc784dcff..7266a1ac11f39e7accf0dcb6ae25a1b43d7af9aa 100644 GIT binary patch delta 7212 zcmY+{4O|q}-Usm6v&*vSlAC}cK`w#_^5QZeFM_!$Dk>N%z9s1(V5p>sSeW>T?{p(Y z{^)QM2n&;IG9WNMSw;XcIc3=S!ROleG;iISlpmYxewUVD+ zWWg&bS>-Vmfe~+0Bd|nJkIHEZbQD=g4oN^1s6?flnn0*Tk5nF$#UQeRFhmu=Ak-amJ;sH)1A5_k!I#88F zUEsh{@-T_)gE~qCAO1p{!J#^;2ZukW+u-PUGCw1-kA_ncz^N91Pgl`#aQ-#w1{dSV zr@zQrf1$Ymm!S|`y-W?@%R;&bu8k%?XOVs9LdgJADgwAPjo?-n^?=*n6zBrJB?;gh zO2Du4s0sAA(jy@Fk;PSHx)4eOPBm2OYQHDVC=QrdJV0cF-=;7y#GNug=mRPP>zb$q zc!(r1xzkjS!+ zl68>SC(kShn(8O(U(c7N3WEog4tiR@u1OH|E2-Aqg&GFAv&~V|jt*~?kjY)tmy8w! z$A!Ey+1*{grwnc1B4v;Z6}h{!cm1ixU1a+!NC5`}$mLmn0&(D|k@CRttyBvdXHzG@ zs&exX*|`Xc2N&m1KKRO=>Hr4r0=SqSo+7&uMTwxZKWzrzl~X-HhuZ*6%+HCeyONT? z-K|srehr}G&)F|VH;NCokk4S;j7*w4I7F*?{4Xj*ag8rE0BtAT1IB&iXBJui2ucR7 zIw}Is#!(~i`ht2ua6bz4!o7KeBoKX(N&ph82~19-M_|S{vJ4TKq@gsBF^WpT$_#1- zIbyLluE;CJ-kvnw+m&strVJd|af8ahE*rIg-5I3w5n08n6barbqfAhFgDL@rY6VS< z3`23DEffR(ww$uSftgeV4i{1z__&9RzPR3}$qE{FQw~7CYJkPt4lXPv(=d@;DWq6% zwVQH>;e!44kZM#PULXaBwR_0LkFVP}(6>M3`Pr{(Es8F_)cHrz%|FBu)58W@Mv4AG ze7nj2k7M=z!E8u8b@_|TCyzXaV=Zr{L=do?HUlj8dNA6PZV$H)U=Bd$AEu-Ld$9n; zS$F6-SRhb0SQ0`$fmq94G#9M7NQL0_FQ@^m{g&>5JRA81iEQHzN)8HfbbN7{AUw5^ zN`ivfrUYt4&Ax5a^B*;q;9&N?FG<0;)aR%KRQOO6*uR(_fod&TMqtw^r!??+1eJnI z=cpNc`CPFDt2dCs>|}uU6I2G2_0$5s8ArMhk#)6EB)It($_xq7YJdKTDpl|6sTGGy zhzz1yJVi08zeiKnA6BV8Y@jw#WO`3BhNAfRS+a)mVos>2cYFg0J&31rM!dYZ|1xAbdGlVO$gwmpeS>q!ri5A(}AE^nPJ4BDbg$}aBi0m_k(g3bVDQI3t z&EV4T;?c;DixdW~PNfX+r9x$(Et^`v^@*f=USu8LP$c*^h%y1LQYGlxM6GrVV{o@8 zQp}h?&nRb1FuS#rs!##iz?~~(w1#MPKg}hp>OFtTQT^48s#SkGO6{soV#zcXY3WC? zz%Y<gFn;wSnjUK2Y1L~rc!V0_MONBFF`zP@vf{;lm8S*4aR14jIr$S_ zUXztWniLZk`?en^3{uxV~32^^kHCE&)J)HEwZ>+n-wdZhZGf-JMKbZ1i< z&^S;j=#xp!p#SOOIU+OfpfG@-86a>emDx)z02`TZF0YFO&o8D-5F104AbtY10?fiN zk2i?{85+t0tB{^x-C=42h-rKgtE+>o;4LT00i}6V4X_!t17w(KKAsy_DHb4RF2E_( zfMXt{fHNn^C0S%&45B!IQXW9dT5u(rI_)L5m$0$qQ9QU&NBQ6fH>!InM616mQx}T3 z1Re`ee0Yo!7l``6*?1{Sqk;v&YRLKp*d5yFHZZOw^Fl1~9!dg(KA{4D;f@2J5b9ni z8tr|B@E`LhjZ;^v9viM6*Kw}IuSX%koogbi0vF*UCxj$0(uGP?E>aT+pHGieZjmJwA3NJ94Mh4-smdv822rbu z(-344g{kyX2EZ;`2C&Pvq=`do6!8w&23}l7#+NbT9kRYGI_X02 z60uf&X-=Fm{*N#^JRPdhB;k2#2eZS-l)?MO0`$uT%Pvw4$XrMY*pWvrOGLIap5j2U z8|A5-p;~arpE^}OBDbYjLTf1=9C4?7m3pcJb?d2Xsi?18kNfdEnU@8#qiZN}8PanC zZ3Z~=dX-kX4eHmEITH^^FG>QR1W^Gv=1Ip@zN2n%d=L4&BC-?bX)ZWtfNMNem$UZ5e2TmBbJe*QcNWPC*B0keMXPK`T1nY;s?_J z7G)_wA~u7z?BbR72QI*qHUl7k%K&zm7J!dgT{izBj0Eb-F~ApECBUarE5Hl5VHLkZ zF#w+tSpXkgRREt~Z2;d7#vDFrE5NRmW2YJ*soTN%E;6ka^|lIi)zi+^mLs{VLx#(P zS24qpY428uWPaQANcq*AV0!7nP#Sjk1=|*#CY(7q^!A==F21wDHri3M-5@ulX^ghN znlx!+9P4^Dy}k7sS(m7p_B0BtxA#b$AgIS|QliE>MwVi$O?0YGyEv zMy6|;7uE7l*EAtIl)c(Dvt(12gRAXKho&tBRdV?l>{=RGZ)G!0>wEE|es#TSsv69v z@@*mnAk!0Fwv<>g0i`}ew zMqaU9H(obqa?)7&(qPRHK1ccbvkp#j_jcV(USXTKLpNfcaa=-@FlFrQBw^0vaq@;P z-I4!3V!f#wpcyBR&CL8knVM}xBdtxiSW>t%T4jX~5Y zs{^U3gK;sSDj&3Ah+(ud22~HJ1FQ1EP2Psl-WXgxpiV$lorJ175mi2!EHre+WYh!d zgjD&Y8oUh&&X|;XK%JN>pPbnTp$F`j)C1eLq;m`IzrlSYeG>OCUoMm6d^B`k7^DdJ z6@UH%w`5E|MQ%|eoWCro6=mT?-17Z(+=>C&+;_I^mSoFNL@>=&1pM}|AHc2d9;L|n zLovCz|0}5#htHRD>mpA{vVtDor#2~S#c|76?uotW+><76?x!0jDYDfUq5Jfa1pLnD zHgTJ7-j?JdRCs3}RMd)=Z!)=;XB62zHI3UUW-D^H8sXSTNx-k&|1P&<&p=7G4l~5N ze6KyGs5RebuiILtK}kR~6ahj|c}# zc}ow^*<8=0|Co8#=J&0x-E-8+gl?=Z)cJTu4=91wFiH`tp`p zr#iXWp$5Bq&vA45-@@GsMCd&{*ED&3c&=&Itwn+t1sY;}w&-}WX|}B5$)?%b8OpnE zoAEQZpwU&5yVVf!JlQlw$9S@7iUX1P)*#Go*8<+MOy|KZUu~7-${@_{2~Rf7zD%BM zn)f?86#?hBKmJ{A^@(%J?qIZ=$a76odz9yz=CJQF9_a8ce{P*VL6Rl4;f{X11<*xt~tEg1d=^&ewKJ0!BJNF^St`%avr)2t=@s;w@Va_HZwoJ?w7(ihHH^ zi6UpAp>x40NkD|_57u)#GTy@)9D%dFG)Yk_zPCDXyAJs)ve|;rQMVcM}brH)bmW8b18l!hO6b2|I`y;fH~0OXe`|Dp!BuH?CupjcYh?9Xm(} zB7~MJ0vej$ozLy>ydG;X1e442;VqxN^)1&kS&tnAJ-j=4?y;dWdG0aa#sj?Pu+R*y zUsWM)rijo%JolKTjOQK`osfG)sPNv!bB~SE@!Vt4tFQ8&(VfH{J0E_KILvRuj3Z%+=~qa4V~Rn6#)^7-tgk?{CxtJ+(?|=qFcP> zo=er-@^Q`BK*ACFv+ca)zESbq_dm$u?spI7R=+s|HxCV+U0fvrp=&o@;U2pC02_!J zA@5_}^5Y*~;~q=iWB0Rp+!HbDuz^G%!da^%Ai~+uC){)UhG5A>U~*39c*~Z&&D=|O zs5Hl^Rj8h_uOZ`>8tD>g~&^IrfCPY@l4Y`I~kdlhzjq^Jkzw^ zV|k`&eGVt_jzhirbA5}$6}eW8;LbBm8~g^(H0_AruS>h5F}2k7ykXSk(cI|qDT+;s zHr#T=ORJ0R>UhIGci%ws%oVG1GR%o9GSl&Q>WZ8kv){Dz*UT@br(~uqPc!E%c_rQ7 zc5Q(nzn@&O+z_JimGhSyp0jgB0MZ2G~D*VM#DR1dCPl-=S^n6 Z^xV|+tQ_;Slvh)huJE^=dCy?g{TuweXU_lt delta 8293 zcma)=dt6ji*T-k}T)Cs5BA||d3d&_Tpdx}|ctM3kK=Xn!C@3fjB9@f~DH#?eu&_gl ziKdB3iMJp%!y=`;go=quHTGC)X_4_5J*m9Ey-)ho^Vd6{&+whK_gZVOwf8#foWq(e zUTd4Z-144FOU+A_%)udPDtz2jsui&+Hy5k$X741 zU`I;k*hD3OG@KeaTBs8UNg;oO#6r7~g<}hq14F~9iQ@s?2Zn7Tvr%Hh7f>1yrl$&y zCOQp_*ji>IvG7U?1x9951`rWXl|ZD4nmLY=)>dMp@+r)A0UPa4RglKcqZVL-BN^=^ z7Ux9~!1P4Qu?yCyQr6QBh?W7=%5j72?IpI*Oi{q%Y2@tSPOsa$J^J|xerA`8(R=v0 z`~rST`jy>PDs3D&+xMio4sQBYDG+B1vCP4Z6&&XJa(;|aN2eTISm7UJh0%IXVN~nj zMrOyJwy$MEOz-j;ucCOz!L0Nwt#Xvu8-7#^Y|5bPz~;f^<|MJV9Vr$-A_c&XI64Sa zPo|r|`>EvVEU~?NC?42%iwc25d#E1xppk9^ABK~uo5T+Hrz8NW76G3wqNBj6rPK~w z7)?IiCDwY2W&qfRV&L-U)Bs#rOC7+~Q1W$=*eyFs1`tvSfURi+e*Ts^fp&NDcLgq! z1?YU8%7H)PsR_7mPxk?BKQi}_n9-NgfNpP7MUP;O+C7p^Lv+7f)>E?e2~eq60pA$; z*LdgAOmsS;RA)C>%IleE1gHh4aT0n%@j1tiXLei_xeyHZ1MclL^w z+Av8e0@(LXAmj@9_s2DwL>6GoCMpM}1W^;mDY_5L z*iB|Hi7m*WG+^mAssNt(l1>B9y;#0fsQa%e40uOFS-`GIR0Y)hNiD$HOfn9@Zp#z_?7c`iK<#1L0o4CStw2Kx*$+hB z`%x5d;sNCWu&V}8#%)0J8FC(ky8n&F0&P#x%0a;zjcOj%aI#>sV%V0EtFNfw7@*s; zw8~f6+FFS2uhDgg8s7ut<`-`nmEcy6L z?D=Cf16Z3y#X#X=YVenAU&_a-7ukTRD!Kn;qboTF^knOj$TvV@#r~8WfWV4>L{*=o z@_;~AeuNq^^TEHV^KUcez(7`4PL@DyYABThdtap{F165o;KOBP4wBeMc9aHuvy&=- ze}&R%;6`zo8O6GZLKS2HKM$cw;MY6U4BV|HZ7_;;JcV&|QdV%VMyr`aRh))AYQgZ% zE;33GjX@N_X}6nl{^AZ!hd^qTB-ZOyvJZjixSpaygqRm1>D_);snk8FCM1x#+fy4R z+;@|6sKh)U(pVsBG_3^2Jfs>ReKuKv1(V2iD0;;?iUF3o&?;b@Kh*-cAJBCm&yn1Q z0pC(AAYZ2fprVux0$ZHvCh*oq@*FO)sxXRIP{{F+>H*|*8`xzeQ<%i|yg^C8d$v>r z?A6dwjw)(b;4?yE``@D(BP4xY1djA~Dj5;T>O-gj6CdVM2XNSle8VMn{2C<#Cw5W^ zaB2)S0$+Yfoj{8N`Hz(JE%&h_3n^`6Ap2@Qm5-FzrD@azw7SuKplvLfBP90q1WE(2 zBNbe_PN#wIO3NZ8_Wdvl1%B918Nkg6R0-U=OwGXU_eeWRV!y;t81P#;WdYcwD&UVF zsYQWtG*0_HiWvPkqrB09?9WcxfeD}$=n5zMs9=pw^)W?p>Pje&Q=d=OoOYhn#_6<& zoX1GabpwqBdcQy`fk9fT0Rq1uD==&*xjupCjR6#+U=@Hmt_4O#&~;#PJ-J0oEIxo@ zIqInZn3_okfsz`!2~6)po?|68w~pcgOA-|Vn5zdeyU}f6Sp}KKp+@Q{3CMezih%rh zItr{_LhXv=GhSlrme33U=EVT)8i3b3r~|+Xz7xcH$pBPJ061#|VAlyi(?3Q8VFA$6 z$^mrQrWmDb-G_)+3 zP8KR*0Gdt~fc{uDS#l^mqf&YIdH6x5Dtg5{ni^s%jnZq>xS1MbCH8I#MF9H_P)@Am zwBJvqGTwW5=c$5m3!lx&qi#W-#ZWmB6uHRKu~6tiZ=EWiB@sEMzz3kJGu^F1-E!e>nRra#7G4k zuhK!_Q%AbVQA(auCD!Ol@ff_J3-IYjR1S39 zrKV}Y8oer)?sMAc$UGg-y^WLxbkCv+z~cg)27LU=p2E}lJqiWjGy{k{N0o}y44|25 zXNb8lV9FWF0;au9RluAYY5@?1ai&m-0G`dD9H0=FD6pY7wF2;IKMNH$mZE^S@+c43 z{V%Ep(2&{yTxHI)akq$|u>gFo1dvh0# z_}n5>3U0s)C<%~!sR+PwM}df~)Se>Qk8Ol)nrhh31rweS6>AEz)9A60Gc005iUAyT zgGI7E(hm*!X?#`GK4m~@lvbm|jBlz?NLEk+#QIVrfICYkfNtYI4})9D0>rJMa^T5G zYU22u?gLY1ka<4vZ%PB=%cugF8bqf#u9c-pETNe~fy5=00ie}Z0%*0(X_CVi{n6yF z@yl*fY7V`)TwI}AwA`?C+z)=BEP%f`rArP;wzxuv@nqu?lm5CsZ8e7EC^9aPSn>xH z0pQ6bXMytIxC0^{4qJiT%VfV0kCC$}YN6y{^i`>Bo`0DCTtR$7^kYX_+$O4_lwLz^ zz|!sHoFOd70OobdjP(9H6A>9VfrI9I1V(0ag z1YE#H4YV*i%JBuY0~ZU(XNknVs-YRcrL$BFw9cXi;IbWcDDYhhX)`4QS4L3@&^C`6 z0oZi{xS#rG12@P5eDe&I14z6H__mhr1J)=q=ZL{H0F_w*;A%V#+`3%0Ou6#{xaDR5 zxQHtOw3%iAFT2`Y@x~Yi@JD6ILM={SJh?4s`1pw<1g(9jfLKcoWIBac#y{*ELX*2%MPjPoDQ&7wQN4lp3!oLZ(LB%;=|pJXlgZbkcY-zK4Z~&$qhTzlWe=X zK8jb?m0!Q4p7vJF)l zrsHyHPko&170%q$DtTl$OSCEAyF^ZhRPyn|8drHwINNE!p^II3Y9z~9`&T$aow3Yk zHUA9fuR-M$?LT2Qb#2Gkg;rrg!$VJVlhD7LJRW?ULHDvBPKpeg6+M+UTJGATd?w{roXRT+VR)m zKUTiU_paj6%9H=T^3A_izR3$i(ZS!ptvvZ3D~qx`B5P*(K8nGW$B^|uF(`vd49->d zb^RMO8^T0Y>Kwn{eG8R$+3*7M#tGo_L9<$`QdKvoY?QyxJ@=gI$ysBQ@Fy`zPWcI? z&g{Vt&P@1PfViHPA^f` zZv+{Q7|&m=eTt9QU0JOSn&~tNsZSa+JxTS{MEppkhWo+t$IG-1^7KpET(AF`#o7Gh zWBb%Z*%YMnAj@u7xp%8}pn9Af)v8_8eH@%c^P?0WTZr;9f4#dL->7q~`>|C!=Ltj- zM7kKExF!7;>ZlvemClJ^pUn-St^h9}TA1rFxWM4PJQ+Li&?JC=i z&?VdO1QbWhM(DyZS~Eho%H#3qKSGC}zk-*e_8MGYULLN)S6{b0+|V?NySB-Nz6M{L zy3%j-O={VDvB6!o%v8_lt5P{Sb7!6=eyIF+^f;_Ljt7Zjn7T&ij?QT>2W-~a82N!N zYtfe1ZQEfe)yNCC>2mFH3%|`Bz*H-5n4<41OHb>EiMpHctRZlSy~kIM;%Z*jqA#j@ zafcyTtLcQQbF{&;?p(EDYs$!ZC6ON zf$&s#Fq}LfK?KFg1CtYMjgc;hfDiB}I7KAeG2x9!_yCWFlSjnKqv8~iF*|rbWPE@} z$0;IAurtQFAVNOCqvYg~%KKsA?T{q~UVbUv!Y7LEJSpV$=@W$f${Dj&?rsm)(|!{J zWoLd83gtsDTI6|{@IEGsfy!Y@np@s_R>*JlRMOn~!fdPj3HM;=D?IGVSJJG$p|Z%` z9T41<9l~P&NhPg=LF=q?t^-`xPZSnM%}QDyRSXgGF=uaK@bO9qi+mo6E~+oZz{!g!_Qp=O;> z=vtg(k?(R39TFc<8?r=RYGaG5kZoceEPUMdz&)#+;Ed=ZJ`@Ap5AG8RuAZAM@^(yk z@0=qBJo|{e)O}tWE#y8A!h}J;^!`?vb%TdH*Tuk~^fsa3d*%d+t{Z|ICaO>^y+2Q!lwK#~kG|e33_c$BCXTZwf;~H144fRFBovz7kF?6$G2uO2RHeH4 zeI?EFJ--tQ=U=}f3@+Sjw8*R%JmiZiRbTx*UnqQ?l7S-Zg>;*gs=N`U6Bajj{em6> z3vaVhmF?9^ns=Q^DBLY779M_UUxDLvgX?FNs{Hd;CCvvj-O)oZ;eAb1CDY9mX)^us zW84G7&6;~)#%)#TA>6~@`C`C%yGWCDw;hKf?2X`ZMO8A7A4OWMPr@nmklt{8Ra7M# zkSNk(1CQhg4}(H7tSY$n-JF2agrbX&s7hw86lpR^gNx709Uh8BRkCocNRy4s+AKVb zye`sYk+G}ML!jt#QB)}#7bntW;}7BDJMWI*`iQDz@f(%2raq`f3-N&Kb)qWS%!l)Z z%h?NJtttdH`o-;lWEd+`#XGKx6^zllX3*N`Y_nn6^&b!w$!XjsblGd`{X9_tt zJ3)9@erY65&l4U}9*TkIFWnOgt4CYWLNMXoB#M%~6s4rKuDVbtylh%642p^tpoR2- zhi;-MS;_kTLZS3mJJehsq&HI(A=`YeT3BouQHmA<3-3=v5we}(N?Px1A0`xbxd#dl z)oa{vdVS&ARumztU2{Pw?ECR3S_me*my04~AO57Ic_cAaC>%MZr1?=uG+GGva8wi} z`z%CB^Tg&GsJVU!PAiI#HLp_AI(L03T1Y>*ULuN+UH)E4>uOwg5!BVs?8FjZ`*+~< zpy+Z)DZ=jqlr(?Xh-=Nu1P^_cBK-LcCCy(RMhXw@Pb+Eu`b|G{5GcAVP>S-8Z>GRuO}9(u&_ST+ zVi85C>D?;Q)ObYUTKkTB_*@jB#(RuNQ{%JS-NG#g^cQJr1{U47$WM5|LpM>Bn!xoU zO-<0RxYo+N5Zo+LgqjiOMOvDX;e#x7cfE{18f0f*y{nwL+u$IN+HIJQXF=Dx_QA%T z?gLDTd8v7s*-QOR3E9h+n}J7%g)zzDP1JOp2}*`_S8hquj`B`cPuEl*z> zkT^d(FYmv0iP Date: Thu, 26 Sep 2013 23:22:23 -0700 Subject: [PATCH 74/77] Upgrade to tzdata2013f. From the release notes: Changes affecting near-future time stamps Tocantins will very likely not observe DST starting this spring. (Thanks to Steffen Thorsen.) Jordan will likely stay at UTC+3 indefinitely, and will not fall back this fall. Palestine will fall back at 00:00, not 01:00. (Thanks to Steffen Thorsen.) (cherry picked from commit 4ced7ef05a2f411c68bbfa82a991d3301eee56e4) Change-Id: Icd4754694fbe3b7c475a63666eeeab36c72908ac --- libc/zoneinfo/tzdata | Bin 560770 -> 552632 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/libc/zoneinfo/tzdata b/libc/zoneinfo/tzdata index 7266a1ac11f39e7accf0dcb6ae25a1b43d7af9aa..5d1d1ebcc8c38a097dcdf5e846fc1b49d1482716 100644 GIT binary patch delta 17466 zcmeI1`CCov-^bUwcO?`#B_V_mLbjcOve<`Cki2i&>>?vI6}w~ zHZo6}({V!PDf9EW-+iCwhv$0!g6GP0<<;x{eD1Yc_g=$$Uu)imTABB2Iz`M385S{2 z@6ohbsKWo86lJQfV(of#pVQK^V(W2ADLY2!CX<~_V=>HyylwPiWEZlttwJkooc{aQ zQ~c}b1sMIvzm7l6zv`_f3q`T$#WNjgh^>>=G(Xls&^TKsF>{yfPcZseThawvdokxJ znR(KD8%#>Kbs~>)j#i5XvW~{2;pLpfvQrdQPA_7sQ99#I7UeVIgUPA9UL<*wZ~0KM z!AJ+$HqWL4#ttH9JH1G~PX@;Bs_WlVz@LRi%NP?>oa*W8ZM!6jGk{O znGqO7c?{mLwG-gQ7`l_f8QMikVGNr?w;03ckWFR1nD~Ia88f<5L}k6z%r}Z+HHp%z zI8avQDq`+%y2r(($H}gW-f9(}N2P|!V#3fx)}?7=l?qmgXL!PLGFNt>*eX@TroNO` zMK4m$k(seOfE=pg&-P*LZ$VLvBYP>GaneZn3_hcrob}?gM!t;PXo_as>_i6{y!QeI zucvc0y)X|a1LIK}TEKXGmNFSUp^(8XF4gto#d!)~yxvQT8E*sV7=xGMHRJsra;?E1 z$7%{>d@WBgjA?$9#ZYbOJ)`s`a&ys(@)`v*erZIp45vwy&8Yr~iWm(&$-Smtc8o6E#3S;jfy2Z$-Nj7zP$)A%q6*736E)Dgf+g%D^uq|fzJJK=6p!)Qh5gJ6U?tIqvpg_j7B8p*5 z|3q2t`Z6=h@@?+AsMAMDsM*unf$SPNiWz2dYor&(ObTYS-b}HK=xLPA*pf~~#e?pR zd3S!KNxY(X#+)F^VWh02Z;Wm3mT0Mi}GRJ4#|?9U>FsWF%QM)r<4x$WtPV#xbtep=8DlFUn&?oF{8N+b8m3 z+?hn-#Ykb?{Yba;O?A2_{$$fkFYfmt?`FjsA^GnKN|h|QLH8t|P9eMI%;!1OxjCQJ zr74nOmqTfc=5@%-XyHH(E%f5o_2k3wx=&FI`>&MF=nzf$jE=X+sij_Y+DX2Q&^;8* z7&V9vGREGa0>=2EoNA!F)Qa`EIdU!wrVtUOxGn0=9sG3Jb+ z*Tr!Cl`s0~6v+6^ieeax6v|>OZcpze?vYz-en{-0V8-gn6w6pwjrr5&JL#y%~stzI0oq~?r6b196G z;Ydl0%v3Tljz*D1JH7a`5_vLCji7N1UY%q{&JoHhhIM=XfTPHZ!ABC#`0E^{6yp}- zQfadB){Dz4$eWQjlp+|{ds8ao_H(+&crcUfIyBW;{5^y^OBVE^NXZY^C{6PFS~7Rg z3rh=f=*W0aKE;S)@Y$Qruq{jZj4F%BsT1GAzL77Z#v+PlxOmb*#?V<*z;G=j=gxZ3 z=r=Mjn$)HR46bD|TA8Vk;XR66y6|bShyoa$y3k@q?;3QB;p;`Oi_NvGUi9;#Kn72b zVeq^x#^BTRp22%?>xTUWGq@#|!5hnF@Vp`hce?xFkI)!=w~A-*4K#eI|+mA92n<%guTFFzz3lNTd;9)YJJ1m5{6KW zL>^@o<9#vQ0`wxW2L&^>?xk49u5pyj`1KhTF%C2(_kMhvJVqKLE{5V6r%zH2<8lZ3 z#<-bB9{u&gCWb=#H`Q7413yvn*%r!`d|MPZfS*Zcs5yhT8pg0mrlevsG57{%5r~>6 zqk0mJW4KPHWQO}J%46`KVLcEnUJRdB6wdJHOO!FVfNn8(n>Imwj@gqpV|*uyU_|bx zR0iLO?lJf>vm3-Wsj}3W!P|^v@G+$^*7qedW9vI|2a2Jj12wEi41+)FEKOf# z$xD8oPLtmq=(S&+Ro)glORl&LLql*eVj1?YDVxC$lOhJ+ZQO_Jg~KS)7!`X^Jj2nF zawJyKH%28F@)*H5P9Y4ZAWD?@M7fO0JL5ulBZ<_UQMCnyG59W<#NfNEDMW93(x@ov zBYEBR7#2=*r{EH`;JdBoNIoH3(YTSJwpFk4wb@c0WiS7L|6G}6bfsLa$vTwJg*oKK z;P4U7;4GNJ;JkN>!4bh`lwNeMP2QvQwrY@~l%Dm&I5XPNx6}V@$AX^>soc~kg6=U| zjv>1+%CZpvaL2h;mvxJ_Uyq+rJ8l@!a^dVsPSJgUUVJ(xvho#}%g9WPn^gRyW$+_5jKLRf5`*t8 zrb+7%dMD%iS2BaY`{Xe=U0YAVf_O3b8$vjPzfGku_`B6D27fiMiNN3K&EPv!L@`ns zeEr^Iq*f=psd}q~Jb8lCs;M3aBByQ}tTvENH>wa>E7|Jufh+7_Oq>v7 z4)4^jg<(?f3FgV|`k1CPHkc!-H#1Ewr#DaiQPDK*lb<&I{wGu9&8FInJWzQwlJbZoBLn&9B;^rG%A=H& zM=B|gR#G0Zq&#X#dE}Dv=q2R}NXk`^9A@ODkiW>~kd#XzDVIf3E{&vI9*~zvQZAFE zTq;SqT#|ChB;~S6%B7Q(%O@$9P*N_Vq%oYAQvM>BQ&KLeq+C`>xwMjUc|l%cNx96D za;YWda!bl3mz2vcDVJVSF2AJw2qfjlz%VNEvycPP{g222Wp39b_`LSl&C=)BodaJ` zBLlwBN`o)_xfcH0t0F>{-eaC0X-(exO; zHe(rlU3xZreew>I>`RK>ZkGF3HbgJMz{c>!XxOBUg(n-vz&CsMf^YeCk;d)HR+lF5 zl**&w+ms6MZJ&F>w?A!ckbOIDH-e|0w}bCIQXal5^(}n&mWF1ze(3j&Z{&|KEBx=~1snJ+ zV|Et|+}`yP19wvD!SBYGF>#-AZ$WAJ{V9d;2P2-tANpT{n|oPn+@?HgmJWaH{tW)N zeH=XBCKsOnE&~2f;U4&td#emQO)>tP8;gO0V^c8jG;K2cSyDgv^VKWN+@};qwSvEx z8V-LMTp9kVUtjp^E-g&5?@f#5@V9j;!QWMMgul1=41fQ=2>v1eu5_dF@j*q6e^EYV zA4fyc@lWv2DVyM5Hr$4PT`(8^ZT3<4_wgGIJWcsAcmw=ruW0G2(iZ2Gsx}%97j@U0 z<+#qaGu*;zHr(PxU3R0QT0R+y0jnDwO|r4n*$!~)!>;hs+iSzitg(WZUFogKKAWip zaNCKs;N|-L1uySs0k>0Qbb)*B3xfZeDB!JEdBNMvtznRTUX!ZB+YVFV?fNM2_TGQPy<0n*WnTvu;~5Ne zto#E5os=E$PM^*2&gLcXF1Js?yPn@>k`ucfNrC(9Tm{F{d2KxTu4)+%g;r?&S!2=%K!uwr$4ex)UzFFaU z1NL2o2ktHlAGqouJZR|)_@J34lk5u~a~VE3Xg_>Nk9~0C(AM!7(3)M+WaF?Z)8NDH z)8WIv1j0wWi-U*UX$>EFEdm~TdVoRB8+D*RJZ$T)@X_Q6A2Yuyd~8&JS@w+^{vAF( zxTSQXI-%o33{3QLLPL1NuO`_zsp=Q_xAJ0JuDb2^VegRyy6cyuFVJaw*v8+@KE zGM+l`eHr-t7oE%sZ)m~&x+ckm7m@MQ-;N{Wsf&Iuh5F*8XQ(fUc?w?|l?z`s%}T@d z9}~O>1IzmrXbNv=g;y+mWsASyv2~E~)KwLc@zgj=WIQ$Q!wS^tpGgKeFaE(Uv*hYC z{_r(_#=sM{wSXsXoB&_Duo8UT9D^Cx|N05dOmbq<5M(@cL+^_4jqMyTVN;_|@Z@?; zHQBe>78y_7Qpz5_<;79dw?6p*Pq}f+Ap5qRMaEONAN~WrWBU=*Q`aEksXJF*FJ9tZ z)8?RI_rwgdoS4=h8BhJ)Z!LUJ%Z{k;Z88gU;EzZFKRW6kgB(9LARm6b8#12yXREXD z6ZLkYp5^k`Ec;HDLdH{1DcSH-|Bgrfw0S!``IHxFWtt|(A3v%N|NBlLJpZI3-KhTa2Qr@eWEV1? z`tLetG!!iPX^<12&iDa;Hu@3#`G_iJIbPTu8BcxD@f-YQ{cWhfs)dZFzW(K;+=2Q= zWTF1%Eh3)!_VE_@yQ>Si&e#9@3nw($_+dXXp89b&GM@S=t~)kRv}``=pC>nje;GRx z{&kR#LC*Wuvm5+-n+EV74eP^yR<)5X6bD2+p?rA>SKrq&%W-kXgy+A|<(0;O#pyzm zELa{u#uHXsFTqREMbxe5r^8E6*$Xc-0vS(~4bIi%Jey7tsM~t&ftRb@A70)$7H(%V z#UT6az9Hg?3Wfa)^7IGhPL%khe7$aunWTVHskWnWQuTG$+3dF*|7 zm66DJqH4frlbq+=xd`=Y&F{jiyWWAMzu~yz(OsIYF zDBShORx$O$u+Pdw4HRvPb_9@SX-lJmK5&CcKwN z1=Nkb9gkqZulzd<`29r26NXpU;e8%v8svuhUe18~pI8SE$h>Bjhu{N@ zLr@QlOn?szn`x4LL4k;PVvtWTJlJ!PtQ*B(w{{p9QZrJMjYF+n;F^kzC$s`&JTdH% z7wW^$<)c30WOaDRKGh)SjohrjLt~Ng#Hi@U@UZYRsE;0vh$qGv5b?y=?q;*R{>3VuIaR_{7r4cw*vbWIPf6^f%Ne-5za{eUs0J!KWPQ36I#nP?O_R*EWJr zi$lf}(`S^2M^5gJ`i#Ig@R@xZYP^3%%<71YC!+MWXqa8|wLxxZPK8%+qk0u?{94v5 z$D+HaD53)dmziQksS!xzmY)EAGL24501)r{+ZX%A#P zv8)p^o``AC8V$>9M`&_GE6O;-SL*u1SH45W6S04L!dG2G#uIU;tH5dhcY~Z4pN5Pl zR>wVruZeLo%khNCC*g@>zu@^V*7n_sfptCap<#U+WIU17@K5-Ls+&!6;>NPc@Qq)Q z@x-S0$ao_8?nu-(=WWtt-C2{N8A-B^$Mm9ujU zaywTK&xT*yu1PnFyiIG(vT*%3WIS=hI17F=)D;tM4MoNixBa}a!#nNUqkgvuGM>0s zqbB@*c?;AZShv#@-rj?k$avym0V19-U&=xKQI0_U@t$4meEt8uBOe3#3CMWjpG9Zk zPp0oQ$PN8FDivNZU>W?W0U1v`YlVy_o_lOJ%f7-&eyG2&TMU1xjO86D;^k{ZJn_oh z3;x>Vi~5_aCMI0}Z_^uN;9YVBG`wHiQ_pN3vXy=Y*0_-CKD2HE$; z6B$o@bwkDz-<+Oa(1X4&_nAQAra5h9*W$(f1gzfL`QKsM+^YOrk3={6(d z=`2=7nxtFKosPQIq#$^yVFTgTeZ1hMdrZ@0Uzw)Jc)GF;0^v3l6x40Yw1(S$sb-LU z<(@gi%isQK#DLxT9~h``Q-YqGEUV0FmJfY+IYjHjzR78y@hZ%|qEyY+a9di~Dz%yM3X2AuJj4P8sa-OKEUH_|sw_swEfW6EF0V1{R(e?9T`vOee^!O z!y!l1JEnX#W%wmqbSU-z;U!Q4zXXa+xHaD*G{a}7MGala>!6a?K_#z)N?r$*ybdaP z9aQopI;iAzP|53{lGi~cuY*cn2bH`IDtR4L j@;a#Gbx_Icp#QuMnyXv%EXRU>EcwSO<5^Cr#54Z|%j3en delta 24104 zcmeI(d0b8F|2FV@uTr6magr%y4z*KFLW~X&PEBWQahsD{TN_77xd&?wEh(RKRW0~E`8%7n&|;L&#hvCkyV;%| z;-_IfG5U_5hA-i#W`~FyLfE(AJquHB7dMCT-xnB&cH5OykA(aDhp<;kzjTTGF8%vx+Vr7+p?l*zvt>J_F2RH2|*d*Dhz4F4=K z3{Ol3^Eit>Fi*>nmn(CRbcTQUIOc74I?jA3L|`bBpc^5Kk_C6HCxFk~o!ymED^0iR{rfu=ql6-P93~wX z!eIoZxVut%5qE8R9lFIk%#EXb?q-L$`a(GN!NwqWcZa3R__cG%DB?;n?(W*^ij?eb z*4C{cgV|h}T#MqzZ^7(vrbuS*OiE!gx=<#=-zc|YX6Cl+GOPe*_L?KLQT}og^gwkmy!W>(M=VNGWnc~#GOd)ONdJ5%% z`S)oVvoMv;GmApW-qWnD7(=z0mCGoMNxV;q44>*Ub6g{*vOLY+9Vv;~ z8co-kT_4D~9MAS4@@LW)Q3S*Lt!Ma)dy6?afbx|$YnP&_DRbo|jblE3rex;xNirC_ zrQ})xU)UB*?gA9a@DGu~xR#?#YjUfI!2l~!70tQU#_=6z**DMTBK})#r$W@;yO35o zfU@`%o{?0n5`T-QQy^1$7)@vRi=M{RFHTPxK7mJNv({=Wbz(TpW;#Eky-aUGFPT9; z3sGu`7RzjIu`)C2%%fV`uMBoSdukx3?(ySGUFpspD_7Uh0%cx1>4iQ$AldcPusa>HT1+jprpbFT`8F+4koOx85IY{jV|f56SjkKuO`&OBZ~Nmi~i&n}U( zzgc_Hjr^JHiWI@Tb))sn#~pNw`P!87HS)2`VON2gN}Yb8aneG|DOp-Pn2bhzC7mVL z#(X91rxsQs8UFUBFkV+Eld0RD+?trR21hA?Y1p2k7=MknGnQtQ#WdYX#hRM6Hh~n# zw0%a?8Qzx0blpHt8Qq6Gn(^17J#}J&^U`c)=p)+8ggMYlYw~Pv)kQW_Z7J zX3|{BX7~&he#QBM7=B3%!w1V?c)t$}ztXD(eh8i6yHy;+_s-)C-;usBd{_5w$zQ|M z6vD)4l)&)sw9`y)5!(v4JYRn7v#wFi296Jp zARpb6t_1knITScV&TaUPoJRguBA5+TX}u(wZZR9HQogoK3N>Yt%`{GuLdndgs$@u# z$+aE-`m97PBELi`iO?XZa`K{1l8rQ*;ib}E$)EI+;S+gw1l^d_3KT6_LFrbqtyJh_ z*3x)!#2g+^G0e%Dl)-f0P9KMam@YS={WNuH+^B=uORO({Kf4; zAzggz985VVL7FR!PD=|MitTFFJm!-x!$%Eeyav-UYdX*H&C9+Uw$)}D44^QksUIaW z0nO+#!#{>ocf7=p2{lnTGm0-yX3|!=&hRmvd+_(`Hu*DiT_}P{h@|xl--K>4e2L}j z$v3Dg)Rf_4j$`;eB{O@9k-;3^N3KC;?Q9Wh!ElOX_?0QlrFxWUO>VvTrV>d3%>C^Y z#XNDN?aZ5a%IamEgkrt)f8&AI+e<4t4u5C5WV zS9XEFyq9zI! zyz30F4V(w^nVymVK(mXzj}SR0$|Z4NhbG^TV$Z)2*7Hl+G@x5d7hlR3iv5~0yx%xx z$T>=8!upWG%#S43L1t}10JUIZ-6&FWm{OQ66)97)liUXLH)kRRFx%WIN|Hv~ne7(J zvQlgaUzy`6klERcrZfC*(0b2D|tXKnY~NNbEsL{ zcZ|9*`yWv>b6_;3GwJ0h+e(FDoOV(WbFe$bFo!2m2E+S(U^4oV*Kq!gs*x^q*|>7z_!weiIWo^2K+Wxjr&>g!LCJ!@pTW8NOzhF??G&KVk{ix+C$I ztwe_Z)^nNR^|aF{Oo$)DeUb%aE`#w>Cf=ZDZ17I`5|=Kgt{ZMw!9Y!4|Ek@RUoje=cdA3q)CMK&^O-Ci1|{eneOKz8 zI&3#OyH3%&m=_sc{#$NzD=^OJcH3fff8E3Av9GVu^P-;-w7iMYYikLk_oT8$pQZJ5 zy_ciWcVcn9Uvp1AxW_j=q@07^zwTXqK#nYZp!;Ec;DeL;AiIBzK}VAH!S@duL)OgG z!w&yp44t>Z81}~uWB9P>W=8n5fyRio!;F!^EsarC1C5A)0s81XW@C(33w_M{cKSHy zO8U5KKKl5l&iaJZg8IZ$&-KVfZ}mw#av77yT+=OaPYvojt8{$Z()Oj}J*I0u@gqvx zmst2LN1^z^h3tyRcbq$yo%|;SzPf}ITP%0-;DkoAZCsQgX6!N4> z<-|gsT&bL3shnh~oM@?>Y^j`ZDNj0U5kgM9;D?-isr&$>@89i+?ii+RgyVPJk} z0~j-@2wc#@1I7k@hEzd=am{}*1V0u!W+E>t`w@B3D*+R7{B8C8J-GP6KDZ<$11>$X z5-y8bYr&J{oA>I1A1lIVBd?sd0y!~oF!HKlQOK+7%z}Tk?FW;J41#N_HnsY(HC&gc zoYfmu;QIHC4Z)8;cjZOiaLp5W<7!J~JV{P z>VhBJN^L{l-tYwS4%0&9oh3IS?|i%!rhc0X(@w_0UH=S+yDb~W;K`l??cv@z-Qd3U zLk+=?{e$ZwADG<^IlW^Screrl9;#j&9&S+(W)yUXN6Npo`uYnz=6KEO#XIo$%WPfn z!*XJ4ItEUjzlwoVOK-!|DQWP`!~^hb%o2F6=PGzUJQ7~0Hw|9wo?r-mTq@ch`Es2J z$d^C=ihQ+55c1W`V0i6g1F1z^&uE5$8#g@Q&7|`1?_C_;Dxr3GzQv zO~^(YCwRB-eR$97F}&aC2+VXo4Ih+RXZ7i3_;0SmhCILc_jC*fvL3JD0U;hGk3@ca za+bkSJXtsoKHV?~J{uhlpU-IxUv%yOUk3sa7VeK6qgxh`}(+6X#jcGU%M&$+e?a;}Uz z$hqhHL+7NT&}DdOm}lC5FmF41m@oJrn7`^nSfI&4STN6V=;~$tC1P@Yz5xS;oevm- zPgwZcQTWT#RnRSUJuGs18g!4J1B)j2hsDN*!Qu<2>Vo$#(X}md$K45PX7)E3Y6|+V>c_^6Xd8Yx!wd zCG--kI%zYk)?x?r?iB-lD#SyxrTHdZ@BynkjmLm*x%tSxSv`?!I*dTBd0`x^^{^wX zy`=}NbF>DmyTA|DTl)+28&Lw*pI_Y&eEbIO1#-jTg^>O07K4^XZQtWb<04;Sld3nY zel%dSJbSF(I0TzNUk-oWl>}Q{+pP;eLCeK6kz1uMLvB6w4;T|wW!xncJM4`Giv z&tT7$$6?UmbFkO!2fEYezbgSwn7$HDTs#3r zhD_1<{K7P8>LNorxoNO2we%ZjNTaG);FQKa;8d5sa9U|UtIwLi8CnUeXUf8vPwMG{ z_n)=N5qb8h;>f>6KSG|f;T!VrV;n4a5|pb{=wE!9S4W?reYy-^_rE_D93`O9NrTieYf^ww7?oWJ@5PEL}E07kt2Fz0JtW zBU>P^XyA&xGN=-AVtF69%Fh|Db}R`0DEi#$!?!T$^Le=D=ry=D^Qj^D`0Lj0LSCP7 z7Wq%h{402}At?oJ9G(i3r^mug?H0q$ArbJeY7=2f(=KpJ-d=EPl~KCj6Ks9a5P7>x zXXNeYN+IufRttG&S_7DRrUu@4>+8SrqSl~#?lFyrkMcqDx`JbEQqYB3#49E^eE`=fLoFrA1BKt8#m zAM&X|gW&1OP2riAt>M|;<>0xBRp9vsPVhp`yzpXqPebtWE@r(#zU=6LeEIk#*?b;}EExOVg;yuRQ9ys{AD@;jOU*zT3D8 zzE6*YAAX+(KPL8upN8~@pQHTYm)5_+{{{saf=}?Zaz*5CEgQ(J3D;)ORMEp~c6n%* zQ?po~T+RvYvoc|hZBL-XMUyW00FHD9*=fstw+s~Htw>%eL-pnopd#ydZ25n9`(PchIg?;99c)yWWiz`z$* zkUO|MM(%Jf6}jWH)5x9DF2T-cHp4CnJ7CvMF|gaXc-VdJNZ6y>c-V8yd|mMIgKGCQ z`20fa)nx<*dY2jp`_$|RbyE-6w{Q)scl}_$cfY{kgC$_dm5;FhDgg%^C}ape?!c&f z$e}CVBM%z%6%L+!0}g3v7<_)Ah4tQpfuR)-!C?)S!{IrTV0a0O)kiboNXKQm;QdCP z2t^+C?_}hNtux^0V>%oYI}nat*BFi)*%FSQgCHgjsNRvw3h&@E?ZY1MV#wj=ZnQ4CMVeIv^h?GZ6W}^I`CyT}!KH1L2|j zmEqydW|(o`{*=vok-9P%8Vs`(b4Zhan}DSQo{ zt-RIhyIt_S>sdqa@y=geihLnE1^MF6NywKj$6Bxd%PSXS!|_~^oU_#!n5$2Fn7dLp=-jY6bjej0<|$do>f>TCpT)_?5PX7s zC%$4Je^x={0^8hS!DDZsYwQJ##{@+L^G5-4m(BuBzKn3{<^;26~(B>w-_=>^@Ho6Z}etZW7Ti4W>^fYHDw5_9WWBs89d7ne7w3|osjFb8iefUT$6K6VOPIW zYYfzX+W|JrRmJKRU+DihFKo1C{)&E(vG?i`3|NZ~eQ2h+q%`TdHzO?=GhL+9N z63R11M+s#=QkNxE*UU~jN~nyQI!dS{HytI^TwiMmHMFpS5-Q+>ffCB=rhyX5d9NXR zlL2pUpo9v&vPVY=wPU%C5^AAEM+r5$uP*zL@Fty`=qRBY^s$yuB^nvB_Za=Pj3G;? zv3GJ9D53WKYoLT$e!@TrHS?j45^BJ4JvA!W-p-!iiQtE-mG~XWtZF5GXEMuoCzbC| zD&M8FJHJz@s+IU1%dBc8e&;f)T8U>s=I^zV=xgDrkOQh#;wh0?)k-`yGTUk;IYlzd zu1+eaN-C#II+&+Ss%j;kI+<0i#8W7cfU1?6Z$MVHQn_ekRVz6RL!R-y zR+{-@jv@Fl%T_C$3qw}5(qC+0kyWjfxWf?q;9AL$=NHSP;xV9VC4D@ys+IhE zBCA@-qdu~#l|)J8HQ#HcwU7LC!H;#eTIoayWK}EuDUdg=u58Hji{#n&FraFs&?+qq}SKwa=d)k+gu zBCA@dM`dJHE7f&DR<%-*7s$3+DdXb>Ngr$)k>{nb-@o+D^-p_R<%;@F37f8>B^JZ$f{O4<&J#aVyl%le8GUK zmEznC!4Fj{jrxSFYNd|1kX5ZzeLu3Ql?o;z+iIoEH&c-xe6N)r-dd>(eppnkw08;y zRIRiu7+KXyk1Q4s+BTM;RUKz zO4@|1YNhG(kyWiU)OyELwNjgTy5N7+QMFQ)aAZ|0xpY9b)k^lyd^z(!tf^Y*YM>$b z0IF8nX+~DH(jr%6RV$5thpcL)&exGutyE(-vZ|FzT+;;~-&QLX{JKk**T1Tj3f)i%nf{&+arM`KQRjuUz3fWdGHS(}r!houk#17;p z-)p6&Mm%yeTdj0rn<4lFs#f}QA+oBKejAIdYNbKlkyWkKvL5pHTFDe>F?Z7iA5hgw z1?nQ(YNd{^iy_-;rOp?=BCA?yi-D|ar3HtORjoAPjv@H?s#fZO8{YR?i93`&bywpB zs#bEJj%=%y`h6OJ9Q?gj3c1+=xxcMeI^16ud;(P~{qZZZs+FcyL{_y@aBgJON__p9 zhBbbM0aYt`=zDb7Q6R29L(M)7jE0qdG zR<)7|Ydq84@3qpsyI$CS-&QLf%ppU%S=CCeSmJ56TFE1O6Rv-))c0Dc^yT^3p^U9o+Lx@$ z1FBkSIo5bTYNbix*sf}&ULBBCtyJF^S=CC#3nSZVC2t3d*${jJTdm~t&=pzLN=M!y zt6FKzb!1g5&Df2sYNcV8`)~5 zvyG5dt(1y2o~CN0_=bky6R28gY$;?_D|NL)R<%;i2guf1iT`1}ZQ)}Wu+>WKzGdoy z4`{2E+W&JDS=CAh)+4K0Y2_SbRV!J-kX5Y|I@=I@JXI^Tz!J||EAjQG^{Nnn7pPjv zsVcIqR?@Su#?$)RYNcbZv0c?lTd>B{RIL<)HQtX}Dg2Ty_yWl5RT512=Cdjs0Dg8=W=yakYLwbGsESmSB`*lMM-SFv5yN}JO-^YyQ4rFg9IG*v5&P1OaT zP}NFZ7bB}$sn$efRV)3{3t81lV!R=E|L5Opr56U4ct2{T^!f&$UuddUN-T*TRILpZmzFR%$79!TYOPsRGt`KWZhX3~aa6N?)_E#M8dnYNg||bw0naQ?=6i!Ey&X zRVyvP8t+G~G$PcH+f}X9z7?{nm3*+qvs1NF{yf-jtCd_n@g0x%&-1-j%6roV-|8Q= z(ykYHfvS}jUqDv1()ex2s#fa05Lwkqjkg$rPoQe0(uAyPC2cgatyU^=&(akGs#ZEw z2U*oh8@uR&52$LT-?7H~Q7a9>8qZGEO0B-&^{Q5?bO%}0O1aXJzt>83RV;6?#w@=JwNf$M@a(L$lCbmj z`%)e+u+>TpZefjQ*U(lg?RkRjs#aR#Xr!)OV?V_%)nU4Qy50W^RZu)vK{4@pLhJ{m zPX5ilq*h#ZTY_#fs< BAPN8g From 3fce4015902bcf8d4257ee7624d463d92453b602 Mon Sep 17 00:00:00 2001 From: Elliott Hughes Date: Sun, 6 Oct 2013 11:35:34 -0700 Subject: [PATCH 75/77] Upgrade to tzdata2013g. From the release notes: Changes affecting current and near-future time stamps Morocco now observes DST from the last Sunday in March to the last Sunday in October, not April to September respectively. (Thanks to Steffen Thorsen.) (cherry picked from commit 40f072192fe080b970502aba8e7d8d01bd4c461a) Change-Id: I247f8cf4ef32ec5d6e6fe3845f9a8977b7e748b9 --- libc/zoneinfo/tzdata | Bin 552632 -> 552701 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/libc/zoneinfo/tzdata b/libc/zoneinfo/tzdata index 5d1d1ebcc8c38a097dcdf5e846fc1b49d1482716..4d511d5721d1a31c922d9f8e2a4a496285aef416 100644 GIT binary patch delta 4693 zcmXY!4^$QBna1yYUf_xsLM-Y9A&i2NDACId5|Us-lqgk9Ku|FvUKJE9F;_)O43=w4 z8%n4_9#Zfes*?sAJYgG>uu5CHOIng*!s(Kh7>>nVvZXEELtIWmBWp;q&pVWJ=J(9= ze)IQz-|xFa&&_3D-dyIXI90sAV!z={HV-@a=W#erW;fRB6}&E$V$fV_m+5JRPGfY)WH zb>S#>ncC0MO8PNKZf=@ysI*L^P##=@()p$~gEn#iLu4_Wp@vFdiL}te+Au7d4WSqg zl}B+cFtzuPPJRLZ0zTpv#fUa;ueHw&unwR0Z%hbM)Z1Oz>$bNK_zrh4Kf|v+H zU0#ly2vhT+mge>4=%I8yj7i4cfPS~BwIZJ^K^^%qdda_FioAk^g>qg2S&xQ=hWcX$ z`e^Q8ntTn3-!QfBqLBPP8p&(uCwF3oJcp!6Q_F!b(tg-xn%`l7{2pe>AH%!I)GnfU zk)akuqm{yp4>H!>wAfIwE~G3rwK|lNC1_*lxl|0%RKU5!Q1Oo-ZHcM<9_2LIsjz4+ zVwi?kad~V>_n6u(_-XbpL_5uIF=EplWhj3mMHiKSsI70T>Lxs`N!RJGHbvP4DZ5(L{hWbe;%4u?8(eV1iG=8{N znA!!TlcVsHZ=;>OgAwu&-1i#lmtJI0{>_Psd*zG{@)}0TVMMHyV>8JjRFP%qTxqEP zOT!pNn~%su>60unfa*m1wp}#M7@sqq`|McuoCNMOLfaJ=$F{E%HaS|pPIE0nu6L1& z`z0psGqwLg%qmm+6glK5YRMAxkS}0zm3U?m?AGCA<5KU zL_w0FzP%m|l&W;}F_wdAno}!~c)zKAi9+(PXe57ye)1ofu_H-q`EE!j}- z1)-eg2rTjph8a2;4wqr5kJ6EDNZ0snwlg&PFh*!v;WlM@XOOL^Fby@~Lm}>TxV)y$XaKp^~+IBiOfJ3WA9*`W@9WosWP#% z$zJ~c@Sv&nqJZ3r2J&(Ak+qn9&`>{IhQx>D#)afnG?IDfCqKr_Lxy@K4oMHQ9^vpk zY--;|Gg*cKhF(p=EKLEt>Cy(po)0UnikYpNFHt4Vy`kOTVyVmlHF({n=!-zPH5*=dpvEcl&PF7g03#1wIvO+`+TT)38om+Ya*!zB4R z^nKE3`TGoYXC3NjD$z@JVrrkc-~)cU6gyIH1vYPxjtl0`S}>DT;IrSjfx^=D`Hawd zlS7r<9GH4?#kmaUydaJw?zaypoRdaMYscc-l}GHlW~XSvniYDfE#qW7=loQA53)jKZUmtVVJM{S5Jp z8Je3&dPWvM-!q2#Ss0pWST}MLX36XD`Xy5Aw_B=}rVoSUCr!sBoj>~W9JCh!;cHVx(+O>0)wd?d%t6)*SUht7_?M}RD?an!3?fyrq{?t`v zJ@rMSwI@E-+Vi1DFSO$H!vAPu=^aEL6TL=6N*1^3utm2K2S|#gytwW1GwGPK#unzzIwqE*Wz$zPf z-a2ygfc348GOPUE8vW_It=7?>?bn|vz2EZxV55F4cZ+`f&u=Vnu5SIg-(Rz;&kwn+(;boecbNwSrRvX#jam4t~((rjhoY-RF9C4sgwiK3E7TbWEzNvN$% zs;DH^Rwh@pMlvirl4V<&X;I0xt<1QnWZhO~-d1K`R4O1URiHSe61K7$qEZo2sfw+v zjIFE=wZUFB}AnyM5Q%E*&deT z9_~nsNI=>|l&xZGV0W{$4E)cBAiO@gW#&)LsFubE<^G=c?DO8U8aKsl9yceqAa21| Ky=Oz}#{M6U8~|qk delta 4645 zcmX|^4_sCC8OHB9{PCg}#U#Bb*fB6sQ7`9UVp3vCG6O}q${(+C)tjPXf`XAyP|?Cd z1s)W(s7{;P2RB!g=-5V!5(_ulsIg)V8#Py~s9`asPh@-E!#4Qb?{l8_cmBTb@BQ6# zzr8%G_3|ud;bGsl!fm=M*4Sy|pVMY5O0n6ae)E!ZoSza1&2ORVd<|)Sx~jd73O_@s zMJwsXAUPMBzph5U0k^-Q>_8~N`vgOI z90}xkP#;j8GTSp@%Sx_ax^h@WOCN03Kz?^T#I+32C+S?o?r5}bx| z8u>Kdw{VK)Vf0#bgz4)350DsUDBnY2m>zies?8Qyfad9;Xb+pAY)2pWc=8p5Og949 z7vplQABg`#@;3}pk(OxM*@Z=!#?G)3qP>o${T7|LRlk}sl}EW;HJ za85YGW%j0!)+mEpMvKirl}vjfmV)} zV=%(nS(~Y=pI?G|rd*|xbfb+SKK>BR3y8YKP);C|oP;WJ9oorY4Bw)wL4QEBCb!}t z7oeIfK!>KQGp}QWCLS@f4CQ5bXR(7uP)Ff;I~hBQQF1R_w;IYk_{be-AZyS?Zmx|q zl#{iQ5m+C2v-;TE$fb9nF;Z7+`_WBP1OF&Pc@FVWy4rXVc{C1~G(3L~&3xEr8%h=u z$a>_@HiAlRHv7^W=XcxcL*nhM&+KWOe!U!?1;=f=`m4{7NO|Q|6y7F>H<1rrdfw8W5zkxilvN6`IxlKHIyO@ zkUv5AJVW^f(#U#LkQ!RadoVc9I#8RhtE>MCH-(Q^Ip0vGqK%x3q4`EoHXm&0#^h(p z>sQ>t8ZVD=Dr=A#qpOETP(^v*I@)8TgNA7y>__whLurMFd>+;0A#{+<7_lNT3+2YW z3wf!3p^hdSo#b(hk}+^C;yUmKe2WZa9~x+08bue4yVfOhF3w`EOIO>zLL<#Nbd$yK zk2REuh>z9POYO*`nGKUnKo3KOmtfa*_1)`8&}9bYTRg?kU$3E;CLIn#T0N0WN1>ss zpS+4D%D}tmW9%S8;&k=1POOSElpqw-gmmHz&203Op$J`UC{H7Kv92auMk!^=H)v+8 z99L-W?uGMqY4Q~E1(cJU(Q>=4?tLEvG&u-g!tVp5S*%#Xw(mqMg?BzkDS8K5y!3%P zURO(cP)T{{EwnMV2SXO4mKw?f$Rq<%McU9#F30dvU9Idx^c`|x4|xF9bskOHM^2 z*@SMg6#jP^$~452JCH}7hDrVkJr?cDWuF8x5c$h>^>=UL)R^=#76eD4p&UaZnS(-d zEt(Q_^)H{HkLEB!Rv5}wtRhoUyh2z1{U@BE`2qS#e}vwxs{zAE9+T3$Wu7&Y6L5um z0?w6OL`IRaQcO7+j~227100ANMEEL0`99LfNK}v|XkEoG>wXNk^*TGG0w4p@U4o2nV=VOtM_nOU_3f zS%c1GYdILDal`dJnJPXq9u4Hf=pyIVuCW%GxHZz)x#WH{lIzgT0d}|lz1DdAy;7z; zG7RRuhU0|I78xPS*2<#oSZ`=GiGh9!zjW*=hO!U|(I=IUa2dEJOMTL=2lET@ zp^v);;N6rs^~(`BQg!vn7f2kF!c;@?p^4_nkI-kCkhQw{)W=x0)=+X$OjGqS&d~VK zPjmDmgszkMoxDz0k9~kr8V{Ojj=zs96rS6eW+<6RNz>Kp_fSrgjTV}k%NU^XB78j; z@4q8$y?o(U(BL0vrSV~qrta_1HpmL%-k_@|K13yr7i}~r|Arw7uMm}PNoKmttSXxN z9<-0i@R&rq4X%A2@e5d!LKBOajk@~lHh4E0N)_s8F1&_LniUwOd9Meq zO)}ymm!X0D5xO?%%wKH=*ZGTxqwr+88S-M&IA*#jTub~jt)ci#Y1%yU08Fw3J!CBG zS=Mkumb{ea)1uWxqL{408FB;q zH*;P13ZWjENXZ_{l+whanJmB+YumX+PMETV5Bw?0Y1X2J=1*^7fMyxOw{oSuinOhA z;tCp`vXy2c2FaJ8<;Yxf=jiI!L#U*QMH{&bLpjETD!$kFY`1?=zWVO5NjJ_fkV(q) zT2EX>Vd>hv`uHPX@Q1-6`E`Jyb+c;{{rv(s61~mZ!80b+lqo~#*d}ZC2>AqJw#${g zM8$ArHH>z%9BU(b^bm*c*Z;IsoUk*Qq$dI4;Sb+F3Jx3PHF1|QbRsD z38%;f=p~D^PusBaf5-cudjw6~qY!=M9)#q|F{{W(6q5ls zL!LlC`7%OxOM@qqYf-veR}XxRW||yap{W^!bB`1#g-k>F9yZk%XrVcW0gK^#r4`c1 z5>${&&`OTSAbAd2o}pyGP4Y(@3aSQeqVc_|3j_# z#x?W7_6yqK4|>g#(|IcFbvGD4Ab zNHa@HEM;0OWokqvJ))8#OPMB1nJQ69m#C!7Ql`yPrp{8PPgGK9Dbr{vQz=>|`IHUG zsin-TsN~jC=2uj5Y$@|>DRV6<`4*L&i%Q-_CHJC|e^IG{sMJAJYC$QpNjlbTq{ zx`;|`EM3gCD Date: Tue, 15 Oct 2013 14:54:02 -0700 Subject: [PATCH 76/77] Remove new aligned memcpy path for cortex-a15. For some reason the new cortex-a15 memcpy code from ARM is really bad for really large copies. This change forces us to go down the old path for all copies. All of my benchmarks show the new version is faster for large copies, but something is going on that I don't understand. Bug: 10838353 Change-Id: I01c16d4a2575e76f4c69862c6f78fd9024eb3fb8 --- libc/arch-arm/cortex-a15/bionic/memcpy_base.S | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libc/arch-arm/cortex-a15/bionic/memcpy_base.S b/libc/arch-arm/cortex-a15/bionic/memcpy_base.S index 015467607..de9e33b64 100644 --- a/libc/arch-arm/cortex-a15/bionic/memcpy_base.S +++ b/libc/arch-arm/cortex-a15/bionic/memcpy_base.S @@ -74,8 +74,10 @@ ENTRY(MEMCPY_BASE) cmp r2, #16 blo .L_copy_less_than_16_unknown_align - cmp r2, #832 - bge .L_check_alignment + // TODO: The aligned copy code is extremely slow copying some large + // buffers so always go through the unaligned path for now. + //cmp r2, #832 + //bge .L_check_alignment .L_copy_unknown_alignment: // Unknown alignment of src and dst. From fbefb252b09634114977dbd1b48dd42bb2629b83 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Tue, 15 Oct 2013 12:10:06 -0700 Subject: [PATCH 77/77] Modify prefetch for krait memcpy. I originally modified the krait mainloop prefetch from cacheline * 8 to * 2. This causes a perf degradation for copies bigger than will fit in the cache. Fixing this back to the original * 8. I tried other multiples, but * 8 is th sweet spot on krait. Bug: 11221806 (cherry picked from commit c3c58fb560fcf1225d4bfb533ba41add8de910e4) Change-Id: I369f81d91ba97a3fcecac84ac57dec921b4758c8 --- libc/arch-arm/krait/bionic/memcpy_base.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/arch-arm/krait/bionic/memcpy_base.S b/libc/arch-arm/krait/bionic/memcpy_base.S index d87a5423b..009c8f92f 100644 --- a/libc/arch-arm/krait/bionic/memcpy_base.S +++ b/libc/arch-arm/krait/bionic/memcpy_base.S @@ -79,7 +79,7 @@ ENTRY(MEMCPY_BASE) 1: /* The main loop copies 64 bytes at a time */ vld1.8 {d0 - d3}, [r1]! vld1.8 {d4 - d7}, [r1]! - pld [r1, #(32*2)] + pld [r1, #(32*8)] subs r2, r2, #64 vst1.8 {d0 - d3}, [r0, :128]! vst1.8 {d4 - d7}, [r0, :128]!