Merge "Fix comment to reflect the actual reality."

The original comment implied that Android 32 bit was the only one
using a bad time_t, but it turns out 32 bit glibc has the same time_t
as Android. Update the comment to reflect this reality.

Change-Id: I72cdd2c677a6521f7b43d4695a52fd8fdc3305bd

.clang-format

@@ -0,0 +1,14 @@
+BasedOnStyle: Google
+AllowShortBlocksOnASingleLine: false
+AllowShortFunctionsOnASingleLine: false
+
+CommentPragmas: NOLINT:.*
+DerivePointerAlignment: false
+IndentWidth: 2
+ContinuationIndentWidth: 2
+PointerAlignment: Left
+TabWidth: 2
+UseTab: Never
+PenaltyExcessCharacter: 32
+
+Cpp11BracedListStyle: false

Android.bp

@@ -0,0 +1 @@
+subdirs = ["*"]

README.md

@@ -194,14 +194,15 @@ The tests are all built from the tests/ directory.
 ### Device tests
 
     $ mma
+    $ adb remount
     $ adb sync
     $ adb shell /data/nativetest/bionic-unit-tests/bionic-unit-tests32
     $ adb shell \
         /data/nativetest/bionic-unit-tests-static/bionic-unit-tests-static32
     # Only for 64-bit targets
-    $ adb shell /data/nativetest/bionic-unit-tests/bionic-unit-tests64
+    $ adb shell /data/nativetest64/bionic-unit-tests/bionic-unit-tests64
     $ adb shell \
-        /data/nativetest/bionic-unit-tests-static/bionic-unit-tests-static64
+        /data/nativetest64/bionic-unit-tests-static/bionic-unit-tests-static64
 
 ### Host tests
 
@@ -256,6 +257,17 @@ First, build and run the host tests as usual (see above).
 The coverage report is now available at `covreport/index.html`.
 
 
+Attaching GDB to the tests
+--------------------------
+
+Bionic's test runner will run each test in its own process by default to prevent
+tests failures from impacting other tests. This also has the added benefit of
+running them in parallel, so they are much faster.
+
+However, this also makes it difficult to run the tests under GDB. To prevent
+each test from being forked, run the tests with the flag `--no-isolate`.
+
+
 LP32 ABI bugs
 -------------
 

benchmarks/Android.mk

@@ -29,7 +29,6 @@ benchmark_cflags := \
     -Wunused \
 
 benchmark_cppflags := \
-    -std=gnu++11 \
 
 benchmarklib_src_files := \
     Benchmark.cpp \

benchmarks/math_benchmark.cpp

@@ -65,6 +65,50 @@ void BM_math_logb::Run(int iters) {
   StopBenchmarkTiming();
 }
 
+BENCHMARK_WITH_ARG(BM_math_isfinite_macro, double)->AT_COMMON_VALS;
+void BM_math_isfinite_macro::Run(int iters, double value) {
+  StartBenchmarkTiming();
+
+  d = 0.0;
+  v = value;
+  for (int i = 0; i < iters; ++i) {
+    d += isfinite(v);
+  }
+
+  StopBenchmarkTiming();
+}
+
+#if defined(__BIONIC__)
+#define test_isfinite __isfinite
+#else
+#define test_isfinite __finite
+#endif
+BENCHMARK_WITH_ARG(BM_math_isfinite, double)->AT_COMMON_VALS;
+void BM_math_isfinite::Run(int iters, double value) {
+  StartBenchmarkTiming();
+
+  d = 0.0;
+  v = value;
+  for (int i = 0; i < iters; ++i) {
+    d += test_isfinite(v);
+  }
+
+  StopBenchmarkTiming();
+}
+
+BENCHMARK_WITH_ARG(BM_math_isinf_macro, double)->AT_COMMON_VALS;
+void BM_math_isinf_macro::Run(int iters, double value) {
+  StartBenchmarkTiming();
+
+  d = 0.0;
+  v = value;
+  for (int i = 0; i < iters; ++i) {
+    d += isinf(v);
+  }
+
+  StopBenchmarkTiming();
+}
+
 BENCHMARK_WITH_ARG(BM_math_isinf, double)->AT_COMMON_VALS;
 void BM_math_isinf::Run(int iters, double value) {
   StartBenchmarkTiming();
@@ -78,6 +122,60 @@ void BM_math_isinf::Run(int iters, double value) {
   StopBenchmarkTiming();
 }
 
+BENCHMARK_WITH_ARG(BM_math_isnan_macro, double)->AT_COMMON_VALS;
+void BM_math_isnan_macro::Run(int iters, double value) {
+  StartBenchmarkTiming();
+
+  d = 0.0;
+  v = value;
+  for (int i = 0; i < iters; ++i) {
+    d += isnan(v);
+  }
+
+  StopBenchmarkTiming();
+}
+
+BENCHMARK_WITH_ARG(BM_math_isnan, double)->AT_COMMON_VALS;
+void BM_math_isnan::Run(int iters, double value) {
+  StartBenchmarkTiming();
+
+  d = 0.0;
+  v = value;
+  for (int i = 0; i < iters; ++i) {
+    d += (isnan)(v);
+  }
+
+  StopBenchmarkTiming();
+}
+
+BENCHMARK_WITH_ARG(BM_math_isnormal_macro, double)->AT_COMMON_VALS;
+void BM_math_isnormal_macro::Run(int iters, double value) {
+  StartBenchmarkTiming();
+
+  d = 0.0;
+  v = value;
+  for (int i = 0; i < iters; ++i) {
+    d += isnormal(v);
+  }
+
+  StopBenchmarkTiming();
+}
+
+#if defined(__BIONIC__)
+BENCHMARK_WITH_ARG(BM_math_isnormal, double)->AT_COMMON_VALS;
+void BM_math_isnormal::Run(int iters, double value) {
+  StartBenchmarkTiming();
+
+  d = 0.0;
+  v = value;
+  for (int i = 0; i < iters; ++i) {
+    d += (__isnormal)(v);
+  }
+
+  StopBenchmarkTiming();
+}
+#endif
+
 BENCHMARK_NO_ARG(BM_math_sin_fast);
 void BM_math_sin_fast::Run(int iters) {
   StartBenchmarkTiming();
@@ -134,3 +232,55 @@ void BM_math_fpclassify::Run(int iters, double value) {
 
   StopBenchmarkTiming();
 }
+
+BENCHMARK_WITH_ARG(BM_math_signbit_macro, double)->AT_COMMON_VALS;
+void BM_math_signbit_macro::Run(int iters, double value) {
+  StartBenchmarkTiming();
+
+  d = 0.0;
+  v = value;
+  for (int i = 0; i < iters; ++i) {
+    d += signbit(v);
+  }
+
+  StopBenchmarkTiming();
+}
+
+BENCHMARK_WITH_ARG(BM_math_signbit, double)->AT_COMMON_VALS;
+void BM_math_signbit::Run(int iters, double value) {
+  StartBenchmarkTiming();
+
+  d = 0.0;
+  v = value;
+  for (int i = 0; i < iters; ++i) {
+    d += (__signbit)(v);
+  }
+
+  StopBenchmarkTiming();
+}
+
+BENCHMARK_WITH_ARG(BM_math_fabs_macro, double)->AT_COMMON_VALS;
+void BM_math_fabs_macro::Run(int iters, double value) {
+  StartBenchmarkTiming();
+
+  d = 0.0;
+  v = value;
+  for (int i = 0; i < iters; ++i) {
+    d += fabs(v);
+  }
+
+  StopBenchmarkTiming();
+}
+
+BENCHMARK_WITH_ARG(BM_math_fabs, double)->AT_COMMON_VALS;
+void BM_math_fabs::Run(int iters, double value) {
+  StartBenchmarkTiming();
+
+  d = 0.0;
+  v = value;
+  for (int i = 0; i < iters; ++i) {
+    d += (fabs)(v);
+  }
+
+  StopBenchmarkTiming();
+}

benchmarks/time_benchmark.cpp

@@ -17,6 +17,7 @@
 #include <sys/syscall.h>
 #include <sys/time.h>
 #include <time.h>
+#include <unistd.h>
 
 #include <benchmark/Benchmark.h>
 

libc/Android.mk

@@ -1,6 +1,6 @@
 LOCAL_PATH := $(call my-dir)
 
-bionic_coverage := false
+bionic_coverage ?= false
 
 # Make everything depend on any changes to included makefiles.
 libc_common_additional_dependencies := $(LOCAL_PATH)/Android.mk
@@ -59,6 +59,7 @@ libc_common_src_files := \
     bionic/system_properties_compat.c \
     stdio/findfp.c \
     stdio/fread.c \
+    stdio/refill.c \
     stdio/snprintf.c\
     stdio/sprintf.c \
     stdio/stdio.c \
@@ -70,12 +71,17 @@ libc_common_src_files := \
 libc_common_src_files += \
     bionic/__FD_chk.cpp \
     bionic/__fgets_chk.cpp \
+    bionic/__fread_chk.cpp \
+    bionic/__fwrite_chk.cpp \
+    bionic/__getcwd_chk.cpp \
     bionic/__memchr_chk.cpp \
     bionic/__memmove_chk.cpp \
     bionic/__memrchr_chk.cpp \
     bionic/__poll_chk.cpp \
     bionic/__pread64_chk.cpp \
     bionic/__pread_chk.cpp \
+    bionic/__pwrite64_chk.cpp \
+    bionic/__pwrite_chk.cpp \
     bionic/__read_chk.cpp \
     bionic/__readlink_chk.cpp \
     bionic/__readlinkat_chk.cpp \
@@ -92,12 +98,14 @@ libc_common_src_files += \
     bionic/__umask_chk.cpp \
     bionic/__vsnprintf_chk.cpp \
     bionic/__vsprintf_chk.cpp \
+    bionic/__write_chk.cpp
 
 libc_bionic_ndk_src_files := \
     bionic/abort.cpp \
     bionic/accept.cpp \
     bionic/accept4.cpp \
     bionic/access.cpp \
+    bionic/arpa_inet.cpp \
     bionic/assert.cpp \
     bionic/atof.cpp \
     bionic/bionic_systrace.cpp \
@@ -208,6 +216,7 @@ libc_bionic_ndk_src_files := \
     bionic/socket.cpp \
     bionic/stat.cpp \
     bionic/statvfs.cpp \
+    bionic/strchrnul.cpp \
     bionic/strerror.cpp \
     bionic/strerror_r.cpp \
     bionic/strsignal.cpp \
@@ -243,10 +252,20 @@ libc_bionic_src_files += bionic/fork.cpp
 # dereferences.
 libc_bionic_src_files += bionic/getauxval.cpp
 
-# These three require getauxval, which isn't available on older platforms.
+# These four require getauxval, which isn't available on older platforms.
 libc_bionic_src_files += bionic/getentropy_linux.c
 libc_bionic_src_files += bionic/sysconf.cpp
 libc_bionic_src_files += bionic/vdso.cpp
+libc_bionic_src_files += bionic/setjmp_cookie.cpp
+
+libc_bionic_src_files += \
+    bionic/__memcpy_chk.cpp \
+    bionic/__memset_chk.cpp \
+    bionic/__strcat_chk.cpp \
+    bionic/__strcpy_chk.cpp \
+    bionic/strchr.cpp \
+    bionic/strnlen.c \
+    bionic/strrchr.cpp \
 
 libc_cxa_src_files := \
     bionic/__cxa_guard.cpp \
@@ -257,30 +276,33 @@ libc_upstream_freebsd_src_files := \
     upstream-freebsd/lib/libc/gen/ldexp.c \
     upstream-freebsd/lib/libc/gen/sleep.c \
     upstream-freebsd/lib/libc/gen/usleep.c \
-    upstream-freebsd/lib/libc/stdlib/abs.c \
     upstream-freebsd/lib/libc/stdlib/getopt_long.c \
-    upstream-freebsd/lib/libc/stdlib/imaxabs.c \
-    upstream-freebsd/lib/libc/stdlib/imaxdiv.c \
-    upstream-freebsd/lib/libc/stdlib/labs.c \
-    upstream-freebsd/lib/libc/stdlib/llabs.c \
     upstream-freebsd/lib/libc/stdlib/qsort.c \
     upstream-freebsd/lib/libc/stdlib/quick_exit.c \
     upstream-freebsd/lib/libc/stdlib/realpath.c \
     upstream-freebsd/lib/libc/string/wcpcpy.c \
     upstream-freebsd/lib/libc/string/wcpncpy.c \
     upstream-freebsd/lib/libc/string/wcscasecmp.c \
+    upstream-freebsd/lib/libc/string/wcscat.c \
+    upstream-freebsd/lib/libc/string/wcschr.c \
+    upstream-freebsd/lib/libc/string/wcscmp.c \
+    upstream-freebsd/lib/libc/string/wcscpy.c \
     upstream-freebsd/lib/libc/string/wcscspn.c \
     upstream-freebsd/lib/libc/string/wcsdup.c \
     upstream-freebsd/lib/libc/string/wcslcat.c \
+    upstream-freebsd/lib/libc/string/wcslen.c \
     upstream-freebsd/lib/libc/string/wcsncasecmp.c \
     upstream-freebsd/lib/libc/string/wcsncat.c \
     upstream-freebsd/lib/libc/string/wcsncmp.c \
     upstream-freebsd/lib/libc/string/wcsncpy.c \
     upstream-freebsd/lib/libc/string/wcsnlen.c \
     upstream-freebsd/lib/libc/string/wcspbrk.c \
+    upstream-freebsd/lib/libc/string/wcsrchr.c \
     upstream-freebsd/lib/libc/string/wcsspn.c \
     upstream-freebsd/lib/libc/string/wcstok.c \
     upstream-freebsd/lib/libc/string/wmemchr.c \
+    upstream-freebsd/lib/libc/string/wmemcmp.c \
+    upstream-freebsd/lib/libc/string/wmemmove.c \
     upstream-freebsd/lib/libc/string/wmemset.c \
 
 libc_upstream_netbsd_src_files := \
@@ -310,6 +332,7 @@ libc_upstream_netbsd_src_files := \
     upstream-netbsd/lib/libc/stdlib/nrand48.c \
     upstream-netbsd/lib/libc/stdlib/_rand48.c \
     upstream-netbsd/lib/libc/stdlib/rand_r.c \
+    upstream-netbsd/lib/libc/stdlib/reallocarr.c \
     upstream-netbsd/lib/libc/stdlib/seed48.c \
     upstream-netbsd/lib/libc/stdlib/srand48.c \
     upstream-netbsd/lib/libc/string/memccpy.c \
@@ -344,11 +367,25 @@ libc_upstream_openbsd_gdtoa_src_files_64 := \
     $(libc_upstream_openbsd_gdtoa_src_files) \
     upstream-openbsd/lib/libc/gdtoa/strtorQ.c \
 
-# These two depend on getentropy_linux.cpp, which isn't in libc_ndk.a.
+# These two depend on getentropy_linux.c, which isn't in libc_ndk.a.
 libc_upstream_openbsd_src_files := \
     upstream-openbsd/lib/libc/crypt/arc4random.c \
     upstream-openbsd/lib/libc/crypt/arc4random_uniform.c \
 
+libc_upstream_openbsd_src_files += \
+    upstream-openbsd/lib/libc/string/memchr.c \
+    upstream-openbsd/lib/libc/string/memmove.c \
+    upstream-openbsd/lib/libc/string/memrchr.c \
+    upstream-openbsd/lib/libc/string/stpcpy.c \
+    upstream-openbsd/lib/libc/string/stpncpy.c \
+    upstream-openbsd/lib/libc/string/strcat.c \
+    upstream-openbsd/lib/libc/string/strcpy.c \
+    upstream-openbsd/lib/libc/string/strlcat.c \
+    upstream-openbsd/lib/libc/string/strlcpy.c \
+    upstream-openbsd/lib/libc/string/strncat.c \
+    upstream-openbsd/lib/libc/string/strncmp.c \
+    upstream-openbsd/lib/libc/string/strncpy.c \
+
 libc_upstream_openbsd_ndk_src_files := \
     upstream-openbsd/lib/libc/compat-43/killpg.c \
     upstream-openbsd/lib/libc/gen/alarm.c \
@@ -391,11 +428,9 @@ libc_upstream_openbsd_ndk_src_files := \
     upstream-openbsd/lib/libc/locale/wctomb.c \
     upstream-openbsd/lib/libc/net/htonl.c \
     upstream-openbsd/lib/libc/net/htons.c \
-    upstream-openbsd/lib/libc/net/inet_addr.c \
     upstream-openbsd/lib/libc/net/inet_lnaof.c \
     upstream-openbsd/lib/libc/net/inet_makeaddr.c \
     upstream-openbsd/lib/libc/net/inet_netof.c \
-    upstream-openbsd/lib/libc/net/inet_network.c \
     upstream-openbsd/lib/libc/net/inet_ntoa.c \
     upstream-openbsd/lib/libc/net/inet_ntop.c \
     upstream-openbsd/lib/libc/net/inet_pton.c \
@@ -455,7 +490,6 @@ libc_upstream_openbsd_ndk_src_files := \
     upstream-openbsd/lib/libc/stdio/puts.c \
     upstream-openbsd/lib/libc/stdio/putwc.c \
     upstream-openbsd/lib/libc/stdio/putwchar.c \
-    upstream-openbsd/lib/libc/stdio/refill.c \
     upstream-openbsd/lib/libc/stdio/remove.c \
     upstream-openbsd/lib/libc/stdio/rewind.c \
     upstream-openbsd/lib/libc/stdio/rget.c \
@@ -489,11 +523,16 @@ libc_upstream_openbsd_ndk_src_files := \
     upstream-openbsd/lib/libc/stdio/wprintf.c \
     upstream-openbsd/lib/libc/stdio/wscanf.c \
     upstream-openbsd/lib/libc/stdio/wsetup.c \
+    upstream-openbsd/lib/libc/stdlib/abs.c \
     upstream-openbsd/lib/libc/stdlib/atoi.c \
     upstream-openbsd/lib/libc/stdlib/atol.c \
     upstream-openbsd/lib/libc/stdlib/atoll.c \
     upstream-openbsd/lib/libc/stdlib/getenv.c \
     upstream-openbsd/lib/libc/stdlib/insque.c \
+    upstream-openbsd/lib/libc/stdlib/imaxabs.c \
+    upstream-openbsd/lib/libc/stdlib/imaxdiv.c \
+    upstream-openbsd/lib/libc/stdlib/labs.c \
+    upstream-openbsd/lib/libc/stdlib/llabs.c \
     upstream-openbsd/lib/libc/stdlib/lsearch.c \
     upstream-openbsd/lib/libc/stdlib/reallocarray.c \
     upstream-openbsd/lib/libc/stdlib/remque.c \
@@ -575,10 +614,6 @@ libc_common_cflags := \
     -D_LIBC=1 \
     -Wall -Wextra -Wunused \
 
-ifneq ($(TARGET_USES_LOGD),false)
-libc_common_cflags += -DTARGET_USES_LOGD
-endif
-
 use_clang := $(USE_CLANG_PLATFORM_BUILD)
 
 # Clang/llvm has incompatible long double (fp128) for x86_64.
@@ -624,7 +659,6 @@ libc_common_conlyflags := \
 
 # Define some common cppflags
 libc_common_cppflags := \
-    -std=gnu++11
 
 # Define some common includes
 # ========================================================
@@ -684,11 +718,13 @@ LOCAL_SRC_FILES += upstream-openbsd/lib/libc/time/wcsftime.c
 
 LOCAL_CFLAGS := $(libc_common_cflags) \
     -fvisibility=hidden \
+    -Wno-unused-parameter \
 
 # Don't use ridiculous amounts of stack.
 LOCAL_CFLAGS += -DALL_STATE
 # Include tzsetwall, timelocal, timegm, time2posix, and posix2time.
 LOCAL_CFLAGS += -DSTD_INSPIRED
+LOCAL_CFLAGS += -DTHREAD_SAFE
 # The name of the tm_gmtoff field in our struct tm.
 LOCAL_CFLAGS += -DTM_GMTOFF=tm_gmtoff
 # Where we store our tzdata.
@@ -789,7 +825,7 @@ LOCAL_SANITIZE := never
 LOCAL_NATIVE_COVERAGE := $(bionic_coverage)
 
 $(eval $(call patch-up-arch-specific-flags,LOCAL_CFLAGS,libc_common_cflags))
-$(eval $(call patch-up-arch-specific-flags,LOCAL_SRC_FILES,libc_freebsd_src_files))
+$(eval $(call patch-up-arch-specific-flags,LOCAL_SRC_FILES_EXCLUDE,libc_freebsd_src_files_exclude))
 include $(BUILD_STATIC_LIBRARY)
 
 
@@ -903,7 +939,7 @@ LOCAL_SANITIZE := never
 LOCAL_NATIVE_COVERAGE := $(bionic_coverage)
 
 $(eval $(call patch-up-arch-specific-flags,LOCAL_CFLAGS,libc_common_cflags))
-$(eval $(call patch-up-arch-specific-flags,LOCAL_SRC_FILES,libc_openbsd_src_files))
+$(eval $(call patch-up-arch-specific-flags,LOCAL_SRC_FILES_EXCLUDE,libc_openbsd_src_files_exclude))
 include $(BUILD_STATIC_LIBRARY)
 
 
@@ -967,6 +1003,7 @@ LOCAL_NATIVE_COVERAGE := $(bionic_coverage)
 
 $(eval $(call patch-up-arch-specific-flags,LOCAL_CFLAGS,libc_common_cflags))
 $(eval $(call patch-up-arch-specific-flags,LOCAL_SRC_FILES,libc_bionic_src_files))
+$(eval $(call patch-up-arch-specific-flags,LOCAL_SRC_FILES_EXCLUDE,libc_bionic_src_files_exclude))
 include $(BUILD_STATIC_LIBRARY)
 
 
@@ -1348,7 +1385,12 @@ LOCAL_CLANG := $(use_clang)
 LOCAL_REQUIRED_MODULES := tzdata
 LOCAL_ADDITIONAL_DEPENDENCIES := \
     $(libc_common_additional_dependencies) \
-    $(LOCAL_PATH)/libc.map \
+    $(LOCAL_PATH)/libc.arm.map \
+    $(LOCAL_PATH)/libc.arm64.map \
+    $(LOCAL_PATH)/libc.mips.map \
+    $(LOCAL_PATH)/libc.mips64.map \
+    $(LOCAL_PATH)/libc.x86.map \
+    $(LOCAL_PATH)/libc.x86_64.map \
 
 # Leave the symbols in the shared library so that stack unwinders can produce
 # meaningful name resolution.
@@ -1375,17 +1417,26 @@ endif
 LOCAL_CXX_STL := none
 LOCAL_SYSTEM_SHARED_LIBRARIES :=
 
+# TODO: This is to work around b/24465209. Remove after root cause is fixed
+LOCAL_LDFLAGS_arm := -Wl,--hash-style=both
+LOCAL_LDFLAGS_x86 := -Wl,--hash-style=both
+
 # Don't re-export new/delete and friends, even if the compiler really wants to.
-LOCAL_LDFLAGS := -Wl,--version-script,$(LOCAL_PATH)/libc.map
+LOCAL_LDFLAGS_arm    += -Wl,--version-script,$(LOCAL_PATH)/libc.arm.map
+LOCAL_LDFLAGS_arm64  += -Wl,--version-script,$(LOCAL_PATH)/libc.arm64.map
+LOCAL_LDFLAGS_mips   += -Wl,--version-script,$(LOCAL_PATH)/libc.mips.map
+LOCAL_LDFLAGS_mips64 += -Wl,--version-script,$(LOCAL_PATH)/libc.mips64.map
+LOCAL_LDFLAGS_x86    += -Wl,--version-script,$(LOCAL_PATH)/libc.x86.map
+LOCAL_LDFLAGS_x86_64 += -Wl,--version-script,$(LOCAL_PATH)/libc.x86_64.map
 
 # We'd really like to do this for all architectures, but since this wasn't done
 # before, these symbols must continue to be exported on LP32 for binary
 # compatibility.
 LOCAL_LDFLAGS_64 := -Wl,--exclude-libs,libgcc.a
 
-# TODO: This is to work around b/19059885. Remove after root cause is fixed
-LOCAL_LDFLAGS_arm := -Wl,--hash-style=both
-LOCAL_LDFLAGS_x86 := -Wl,--hash-style=both
+# Unfortunately --exclude-libs clobbers our version script, so we have to
+# prevent the build system from using this flag.
+LOCAL_NO_EXCLUDE_LIBS := true
 
 $(eval $(call patch-up-arch-specific-flags,LOCAL_CFLAGS,libc_common_cflags))
 $(eval $(call patch-up-arch-specific-flags,LOCAL_SRC_FILES,libc_arch_dynamic_src_files))
@@ -1426,8 +1477,8 @@ LOCAL_C_INCLUDES := \
     $(libc_common_c_includes) \
 
 LOCAL_SRC_FILES := \
+    bionic/debug_backtrace.cpp \
     bionic/debug_mapinfo.cpp \
-    bionic/debug_stacktrace.cpp \
     bionic/libc_logging.cpp \
     bionic/malloc_debug_leak.cpp \
     bionic/malloc_debug_check.cpp \
@@ -1444,12 +1495,17 @@ LOCAL_SYSTEM_SHARED_LIBRARIES :=
 # Only need this for arm since libc++ uses its own unwind code that
 # doesn't mix with the other default unwind code.
 LOCAL_STATIC_LIBRARIES_arm := libunwind_llvm
+LOCAL_LDFLAGS_arm := -Wl,--exclude-libs,libunwind_llvm.a
 LOCAL_STATIC_LIBRARIES += libc++abi
 LOCAL_ALLOW_UNDEFINED_SYMBOLS := true
 
 # Don't re-export new/delete and friends, even if the compiler really wants to.
 LOCAL_LDFLAGS := -Wl,--version-script,$(LOCAL_PATH)/version_script.txt
 
+# Unfortunately --exclude-libs clobbers our version script, so we have to
+# prevent the build system from using this flag.
+LOCAL_NO_EXCLUDE_LIBS := true
+
 # Don't install on release build
 LOCAL_MODULE_TAGS := eng debug
 LOCAL_SANITIZE := never
@@ -1490,6 +1546,10 @@ LOCAL_SYSTEM_SHARED_LIBRARIES :=
 # Don't re-export new/delete and friends, even if the compiler really wants to.
 LOCAL_LDFLAGS := -Wl,--version-script,$(LOCAL_PATH)/version_script.txt
 
+# Unfortunately --exclude-libs clobbers our version script, so we have to
+# prevent the build system from using this flag.
+LOCAL_NO_EXCLUDE_LIBS := true
+
 # Don't install on release build
 LOCAL_MODULE_TAGS := eng debug
 LOCAL_SANITIZE := never
@@ -1514,7 +1574,7 @@ LOCAL_C_INCLUDES := $(libc_common_c_includes) bionic/libstdc++/include
 LOCAL_CFLAGS := $(libc_common_cflags)
 LOCAL_CPPFLAGS := $(libc_common_cppflags)
 
-# TODO: This is to work around b/19059885. Remove after root cause is fixed
+# TODO: This is to work around b/24465209. Remove after root cause is fixed
 LOCAL_LDFLAGS_arm := -Wl,--hash-style=both
 LOCAL_LDFLAGS_x86 := -Wl,--hash-style=both
 

libc/NOTICE

@@ -307,22 +307,6 @@ SUCH DAMAGE.
 
 -------------------------------------------------------------------
 
-Copyright (C) 2009 The Android Open Source Project
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
--------------------------------------------------------------------
-
 Copyright (C) 2009 The Android Open Source Project
 All rights reserved.
 
@@ -3921,35 +3905,6 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
 -------------------------------------------------------------------
 
-Copyright (c) 2009
-     MIPS Technologies, Inc., California.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-1. Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-   notice, this list of conditions and the following disclaimer in the
-   documentation and/or other materials provided with the distribution.
-3. Neither the name of the MIPS Technologies, Inc., nor the names of its
-   contributors may be used to endorse or promote products derived from
-   this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE MIPS TECHNOLOGIES, INC. ``AS IS'' AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED.  IN NO EVENT SHALL THE MIPS TECHNOLOGIES, INC. BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGE.
-
--------------------------------------------------------------------
-
 Copyright (c) 2009 David Schultz <das@FreeBSD.org>
 All rights reserved.
 
@@ -4431,6 +4386,64 @@ Copyright (c) 2012, Linaro Limited
 
 -------------------------------------------------------------------
 
+Copyright (c) 2012-2015
+     MIPS Technologies, Inc., California.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the name of the MIPS Technologies, Inc., nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE MIPS TECHNOLOGIES, INC. ``AS IS'' AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL THE MIPS TECHNOLOGIES, INC. BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.
+
+-------------------------------------------------------------------
+
+Copyright (c) 2013
+     MIPS Technologies, Inc., California.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the name of the MIPS Technologies, Inc., nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE MIPS TECHNOLOGIES, INC. ``AS IS'' AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL THE MIPS TECHNOLOGIES, INC. BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.
+
+-------------------------------------------------------------------
+
 Copyright (c) 2013 ARM Ltd
 All rights reserved.
 
@@ -4504,6 +4517,35 @@ POSSIBILITY OF SUCH DAMAGE.
 
 -------------------------------------------------------------------
 
+Copyright (c) 2014
+     Imagination Technologies Limited.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the name of the MIPS Technologies, Inc., nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY IMAGINATION TECHNOLOGIES LIMITED ``AS IS'' AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL IMAGINATION TECHNOLOGIES LIMITED BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.
+
+-------------------------------------------------------------------
+
 Copyright (c) 2014 Theo de Raadt <deraadt@openbsd.org>
 Copyright (c) 2014 Bob Beck <beck@obtuse.com>
 

libc/SYSCALLS.TXT

@@ -77,7 +77,6 @@ int prlimit64(pid_t, int, struct rlimit64*, const struct rlimit64*)  arm,mips,x8
 int     setgroups:setgroups32(int, const gid_t*)   arm,x86
 int     setgroups:setgroups(int, const gid_t*)     arm64,mips,mips64,x86_64
 int     setpgid(pid_t, pid_t)  all
-pid_t   vfork(void)  arm
 int     setregid:setregid32(gid_t, gid_t)  arm,x86
 int     setregid:setregid(gid_t, gid_t)    arm64,mips,mips64,x86_64
 int     chroot(const char*)  all
@@ -95,6 +94,16 @@ ssize_t     pread64(int, void*, size_t, off64_t) arm,mips,x86
 ssize_t     pread64|pread(int, void*, size_t, off_t) arm64,mips64,x86_64
 ssize_t     pwrite64(int, void*, size_t, off64_t) arm,mips,x86
 ssize_t     pwrite64|pwrite(int, void*, size_t, off_t) arm64,mips64,x86_64
+
+# On LP32, preadv/pwritev don't use off64_t --- they use pairs of 32-bit
+# arguments to avoid problems on architectures like ARM where 64-bit arguments
+# must be in a register pair starting with an even-numbered register.
+# See linux/fs/read_write.c and https://lwn.net/Articles/311630/.
+ssize_t     __preadv64:preadv(int, const struct iovec*, int, long, long) arm,mips,x86
+ssize_t     preadv|preadv64(int, const struct iovec*, int, off_t) arm64,mips64,x86_64
+ssize_t     __pwritev64:pwritev(int, const struct iovec*, int, long, long) arm,mips,x86
+ssize_t     pwritev|pwritev64(int, const struct iovec*, int, off_t) arm64,mips64,x86_64
+
 int         ___close:close(int)  all
 pid_t       __getpid:getpid()  all
 int         munmap(void*, size_t)  all
@@ -333,7 +342,7 @@ int     __set_tls:set_thread_area(void*) mips,mips64
 int     __set_thread_area:set_thread_area(void*) x86
 
 # vdso stuff.
-int clock_gettime(clockid_t, timespec*)                 arm,mips,mips64,x86
-int __clock_gettime:clock_gettime(clockid_t, timespec*) arm64,x86_64
-int gettimeofday(timeval*, timezone*)                   arm,mips,mips64,x86
-int __gettimeofday:gettimeofday(timeval*, timezone*)    arm64,x86_64
+int clock_gettime(clockid_t, timespec*)                 arm,mips,mips64
+int __clock_gettime:clock_gettime(clockid_t, timespec*) arm64,x86,x86_64
+int gettimeofday(timeval*, timezone*)                   arm,mips,mips64
+int __gettimeofday:gettimeofday(timeval*, timezone*)    arm64,x86,x86_64

libc/arch-arm/arm.mk

@@ -1,33 +1,19 @@
 # 32-bit arm.
 
-#
-# Default implementations of functions that are commonly optimized.
-#
-
 libc_bionic_src_files_arm += \
-    bionic/strchr.cpp \
-    bionic/strnlen.c \
-    bionic/strrchr.cpp \
+    arch-arm/generic/bionic/memcmp.S \
+    arch-arm/generic/bionic/memcpy.S \
+    arch-arm/generic/bionic/memset.S \
+    arch-arm/generic/bionic/strcmp.S \
+    arch-arm/generic/bionic/strcpy.S \
+    arch-arm/generic/bionic/strlen.c \
 
-libc_freebsd_src_files_arm += \
-    upstream-freebsd/lib/libc/string/wcscat.c \
-    upstream-freebsd/lib/libc/string/wcschr.c \
-    upstream-freebsd/lib/libc/string/wcscmp.c \
-    upstream-freebsd/lib/libc/string/wcscpy.c \
-    upstream-freebsd/lib/libc/string/wcslen.c \
-    upstream-freebsd/lib/libc/string/wcsrchr.c \
-    upstream-freebsd/lib/libc/string/wmemcmp.c \
-    upstream-freebsd/lib/libc/string/wmemmove.c \
+libc_bionic_src_files_exclude_arm += \
+    bionic/__memcpy_chk.cpp \
+    bionic/__memset_chk.cpp \
 
-libc_openbsd_src_files_arm += \
-    upstream-openbsd/lib/libc/string/memchr.c \
-    upstream-openbsd/lib/libc/string/memrchr.c \
-    upstream-openbsd/lib/libc/string/stpncpy.c \
-    upstream-openbsd/lib/libc/string/strlcat.c \
-    upstream-openbsd/lib/libc/string/strlcpy.c \
-    upstream-openbsd/lib/libc/string/strncat.c \
-    upstream-openbsd/lib/libc/string/strncmp.c \
-    upstream-openbsd/lib/libc/string/strncpy.c \
+libc_openbsd_src_files_exclude_arm += \
+    upstream-openbsd/lib/libc/string/strcpy.c \
 
 #
 # Inherently architecture-specific code.
@@ -39,10 +25,11 @@ libc_bionic_src_files_arm += \
     arch-arm/bionic/__bionic_clone.S \
     arch-arm/bionic/_exit_with_stack_teardown.S \
     arch-arm/bionic/libgcc_compat.c \
-    arch-arm/bionic/libgcc_protect_unwind.c \
+    arch-arm/bionic/popcount_tab.c \
     arch-arm/bionic/__restore.S \
     arch-arm/bionic/setjmp.S \
     arch-arm/bionic/syscall.S \
+    arch-arm/bionic/vfork.S \
 
 libc_arch_static_src_files_arm := arch-arm/bionic/exidx_static.c
 libc_arch_dynamic_src_files_arm := arch-arm/bionic/exidx_dynamic.c
@@ -51,6 +38,7 @@ libc_arch_dynamic_src_files_arm := arch-arm/bionic/exidx_dynamic.c
 ifeq ($(strip $(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)),)
   $(warning TARGET_$(my_2nd_arch_prefix)ARCH is arm, but TARGET_$(my_2nd_arch_prefix)CPU_VARIANT is not defined)
 endif
+ifneq ($(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT),generic)
 cpu_variant_mk := $(LOCAL_PATH)/arch-arm/$(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT)/$(TARGET_$(my_2nd_arch_prefix)CPU_VARIANT).mk
 ifeq ($(wildcard $(cpu_variant_mk)),)
 $(error "TARGET_$(my_2nd_arch_prefix)CPU_VARIANT not set or set to an unknown value. Possible values are cortex-a7, cortex-a8, cortex-a9, cortex-a15, krait, denver. Use generic for devices that do not have a CPU similar to any of the supported cpu variants.")
@@ -59,6 +47,7 @@ include $(cpu_variant_mk)
 libc_common_additional_dependencies += $(cpu_variant_mk)
 
 cpu_variant_mk :=
+endif
 
 
 libc_crt_target_cflags_arm := \

libc/arch-arm/bionic/__restore.S

@@ -34,7 +34,9 @@
 // __restore_rt (but covered by the .fnstart/.fnend) so that although they're
 // not inside the functions from objdump's point of view, an unwinder that
 // blindly looks at the previous instruction (but is then smart enough to check
-// the DWARF information to find out where it landed) gets the right answer.
+// the unwind information to find out where it landed) gets the right answer.
+// Make sure not to have both DWARF and ARM unwind information, so only
+// use the ARM unwind information.
 
 // We need to place .fnstart ourselves (but we may as well keep the free .fnend).
 #undef __bionic_asm_custom_entry
@@ -44,18 +46,18 @@
   .save {r0-r15}
   .pad #32
   nop
-ENTRY_PRIVATE(__restore)
+ENTRY_PRIVATE_NO_DWARF(__restore)
   // This function must have exactly this instruction sequence.
   mov r7, #__NR_sigreturn
   swi #0
-END(__restore)
+END_NO_DWARF(__restore)
 
   .fnstart
   .save {r0-r15}
   .pad #160
   nop
-ENTRY_PRIVATE(__restore_rt)
+ENTRY_PRIVATE_NO_DWARF(__restore_rt)
   // This function must have exactly this instruction sequence.
   mov r7, #__NR_rt_sigreturn
   swi #0
-END(__restore_rt)
+END_NO_DWARF(__restore_rt)

libc/arch-arm/bionic/libgcc_protect_unwind.c

@@ -1,93 +0,0 @@
-/*
- * Copyright (C) 2015 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// TODO: This file should go away once unwinder migration to libc++.so is complete.
-
-extern char _Unwind_Backtrace __attribute((visibility("protected")));
-extern char __gnu_Unwind_Find_exidx __attribute((visibility("protected")));
-extern char __gnu_Unwind_Restore_VFP_D __attribute((visibility("protected")));
-extern char __gnu_Unwind_Restore_VFP __attribute((visibility("protected")));
-extern char __gnu_Unwind_Restore_VFP_D_16_to_31 __attribute((visibility("protected")));
-extern char __gnu_Unwind_Restore_WMMXD __attribute((visibility("protected")));
-extern char __gnu_Unwind_Restore_WMMXC __attribute((visibility("protected")));
-extern char _Unwind_GetCFA __attribute((visibility("protected")));
-extern char __gnu_Unwind_RaiseException __attribute((visibility("protected")));
-extern char __gnu_Unwind_ForcedUnwind __attribute((visibility("protected")));
-extern char __gnu_Unwind_Resume __attribute((visibility("protected")));
-extern char __gnu_Unwind_Resume_or_Rethrow __attribute((visibility("protected")));
-extern char _Unwind_Complete __attribute((visibility("protected")));
-extern char _Unwind_DeleteException __attribute((visibility("protected")));
-extern char _Unwind_VRS_Get __attribute((visibility("protected")));
-extern char _Unwind_VRS_Set __attribute((visibility("protected")));
-extern char __gnu_Unwind_Backtrace __attribute((visibility("protected")));
-extern char _Unwind_VRS_Pop __attribute((visibility("protected")));
-extern char __gnu_Unwind_Save_VFP_D __attribute((visibility("protected")));
-extern char __gnu_Unwind_Save_VFP __attribute((visibility("protected")));
-extern char __gnu_Unwind_Save_VFP_D_16_to_31 __attribute((visibility("protected")));
-extern char __gnu_Unwind_Save_WMMXD __attribute((visibility("protected")));
-extern char __gnu_Unwind_Save_WMMXC __attribute((visibility("protected")));
-extern char ___Unwind_RaiseException __attribute((visibility("protected")));
-extern char _Unwind_RaiseException __attribute((visibility("protected")));
-extern char ___Unwind_Resume __attribute((visibility("protected")));
-extern char _Unwind_Resume __attribute((visibility("protected")));
-extern char ___Unwind_Resume_or_Rethrow __attribute((visibility("protected")));
-extern char _Unwind_Resume_or_Rethrow __attribute((visibility("protected")));
-extern char ___Unwind_ForcedUnwind __attribute((visibility("protected")));
-extern char _Unwind_ForcedUnwind __attribute((visibility("protected")));
-extern char ___Unwind_Backtrace __attribute((visibility("protected")));
-extern char _Unwind_GetRegionStart __attribute((visibility("protected")));
-extern char _Unwind_GetLanguageSpecificData __attribute((visibility("protected")));
-extern char _Unwind_GetDataRelBase __attribute((visibility("protected")));
-extern char _Unwind_GetTextRelBase __attribute((visibility("protected")));
-
-void* __bionic_libgcc_unwind_symbols[] = {
-    &_Unwind_Backtrace,
-    &__gnu_Unwind_Find_exidx,
-    &__gnu_Unwind_Restore_VFP_D,
-    &__gnu_Unwind_Restore_VFP,
-    &__gnu_Unwind_Restore_VFP_D_16_to_31,
-    &__gnu_Unwind_Restore_WMMXD,
-    &__gnu_Unwind_Restore_WMMXC,
-    &_Unwind_GetCFA,
-    &__gnu_Unwind_RaiseException,
-    &__gnu_Unwind_ForcedUnwind,
-    &__gnu_Unwind_Resume,
-    &__gnu_Unwind_Resume_or_Rethrow,
-    &_Unwind_Complete,
-    &_Unwind_DeleteException,
-    &_Unwind_VRS_Get,
-    &_Unwind_VRS_Set,
-    &__gnu_Unwind_Backtrace,
-    &_Unwind_VRS_Pop,
-    &__gnu_Unwind_Save_VFP_D,
-    &__gnu_Unwind_Save_VFP,
-    &__gnu_Unwind_Save_VFP_D_16_to_31,
-    &__gnu_Unwind_Save_WMMXD,
-    &__gnu_Unwind_Save_WMMXC,
-    &___Unwind_RaiseException,
-    &_Unwind_RaiseException,
-    &___Unwind_Resume,
-    &_Unwind_Resume,
-    &___Unwind_Resume_or_Rethrow,
-    &_Unwind_Resume_or_Rethrow,
-    &___Unwind_ForcedUnwind,
-    &_Unwind_ForcedUnwind,
-    &___Unwind_Backtrace,
-    &_Unwind_GetRegionStart,
-    &_Unwind_GetLanguageSpecificData,
-    &_Unwind_GetDataRelBase,
-    &_Unwind_GetTextRelBase,
-};

libc/arch-arm/bionic/popcount_tab.c

@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+// Export this to maintain ABI compatibilty with libgcc, since compiler-rt
+// doesn't use a table-driven implementation of __popcount.
+const unsigned char __popcount_tab[256] = {
+  0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3,
+  3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4,
+  3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4,
+  4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5,
+  3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2,
+  2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5,
+  4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5,
+  5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+  3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5,
+  5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
+};

libc/arch-arm/bionic/setjmp.S

@@ -52,7 +52,7 @@
 // Current layout (may change in the future):
 //
 // word   name            description
-// 0      magic        magic number
+// 0      sigflag/cookie  setjmp cookie in top 31 bits, signal mask flag in low bit
 // 1      sigmask         signal mask (not used with _setjmp / _longjmp)
 // 2      float_base      base of float registers (d8 to d15)
 // 18     float_state     floating-point status and control register
@@ -80,33 +80,82 @@ ENTRY(_setjmp)
   b sigsetjmp
 END(_setjmp)
 
+#define MANGLE_REGISTERS 1
+.macro m_mangle_registers reg
+#if MANGLE_REGISTERS
+  eor r4, r4, \reg
+  eor r5, r5, \reg
+  eor r6, r6, \reg
+  eor r7, r7, \reg
+  eor r8, r8, \reg
+  eor r9, r9, \reg
+  eor r10, r10, \reg
+  eor r11, r11, \reg
+  eor r12, r12, \reg
+  eor r13, r13, \reg
+  eor r14, r14, \reg
+#endif
+.endm
+
+.macro m_unmangle_registers reg
+  m_mangle_registers \reg
+.endm
+
 // int sigsetjmp(sigjmp_buf env, int save_signal_mask);
 ENTRY(sigsetjmp)
-  // Record whether or not we're saving the signal mask.
+  stmfd sp!, {r0, lr}
+  .cfi_def_cfa_offset 8
+  .cfi_rel_offset r0, 0
+  .cfi_rel_offset lr, 4
+
+  mov r0, r1
+  bl __bionic_setjmp_cookie_get
+  mov r1, r0
+
+  ldmfd sp, {r0}
+
+  // Save the setjmp cookie for later.
+  bic r2, r1, #1
+  stmfd sp!, {r2}
+  .cfi_adjust_cfa_offset 4
+
+  // Record the setjmp cookie and whether or not we're saving the signal mask.
   str r1, [r0, #(_JB_SIGFLAG * 4)]
 
   // Do we need to save the signal mask?
-  teq r1, #0
+  tst r1, #1
   beq 1f
 
-  // Get current signal mask.
-  stmfd sp!, {r0, r14}
-  .cfi_def_cfa_offset 8
-  .cfi_rel_offset r0, 0
-  .cfi_rel_offset r14, 4
-  mov r0, #0
-  bl sigblock
-  mov r1, r0
-  ldmfd sp!, {r0, r14}
-  .cfi_def_cfa_offset 0
+  // Align the stack.
+  sub sp, #4
+  .cfi_adjust_cfa_offset 4
 
-  // Save the signal mask.
-  str r1, [r0, #(_JB_SIGMASK * 4)]
+  // Save the current signal mask.
+  add r2, r0, #(_JB_SIGMASK * 4)
+  mov r0, #2 // SIG_SETMASK
+  mov r1, #0
+  bl sigprocmask
+
+  // Unalign the stack.
+  add sp, #4
+  .cfi_adjust_cfa_offset -4
 
 1:
+  ldmfd sp!, {r2}
+  .cfi_adjust_cfa_offset -4
+  ldmfd sp!, {r0, lr}
+  .cfi_adjust_cfa_offset -8
+  .cfi_restore r0
+  .cfi_restore lr
+
   // Save core registers.
   add r1, r0, #(_JB_CORE_BASE * 4)
-  stmia r1, {r4-r14}
+  m_mangle_registers r2
+
+  // ARM deprecates using sp in the register list for stmia.
+  stmia r1, {r4-r12, lr}
+  str sp, [r1, #(10 * 4)]
+  m_unmangle_registers r2
 
   // Save floating-point registers.
   add r1, r0, #(_JB_FLOAT_BASE * 4)
@@ -122,29 +171,30 @@ END(sigsetjmp)
 
 // void siglongjmp(sigjmp_buf env, int value);
 ENTRY(siglongjmp)
-  // Do we need to restore the signal mask?
-  ldr r2, [r0, #(_JB_SIGFLAG * 4)]
-  teq r2, #0
-  beq 1f
-
-  // Restore the signal mask.
-  stmfd sp!, {r0, r1, r14}
+  stmfd sp!, {r0, r1, lr}
   .cfi_def_cfa_offset 12
   .cfi_rel_offset r0, 0
   .cfi_rel_offset r1, 4
-  .cfi_rel_offset r14, 8
-  sub sp, sp, #4 // Align the stack.
-  .cfi_adjust_cfa_offset 4
+  .cfi_rel_offset lr, 8
 
+  // Fetch the signal flag.
+  ldr r1, [r0, #(_JB_SIGFLAG * 4)]
+
+  // Do we need to restore the signal mask?
+  ands r1, r1, #1
+  beq 1f
+
+  // Restore the signal mask.
   ldr r0, [r0, #(_JB_SIGMASK * 4)]
   bl sigsetmask
 
-  add sp, sp, #4 // Unalign the stack.
-  .cfi_adjust_cfa_offset -4
-  ldmfd sp!, {r0, r1, r14}
-  .cfi_def_cfa_offset 0
-
 1:
+  ldmfd sp!, {r0, r1, lr}
+  .cfi_adjust_cfa_offset -12
+  .cfi_restore r0
+  .cfi_restore r1
+  .cfi_restore lr
+
   // Restore floating-point registers.
   add r2, r0, #(_JB_FLOAT_BASE * 4)
   vldmia r2, {d8-d15}
@@ -154,16 +204,27 @@ ENTRY(siglongjmp)
   fmxr fpscr, r2
 
   // Restore core registers.
+  ldr r3, [r0, #(_JB_SIGFLAG * 4)]
+  bic r3, r3, #1
   add r2, r0, #(_JB_CORE_BASE * 4)
-  ldmia r2, {r4-r14}
 
-  // Validate sp and r14.
-  teq sp, #0
-  teqne r14, #0
-  bleq longjmperror
+  // ARM deprecates using sp in the register list for ldmia.
+  ldmia r2, {r4-r12, lr}
+  ldr sp, [r2, #(10 * 4)]
+  m_unmangle_registers r3
+
+  // Save the return value/address and check the setjmp cookie.
+  stmfd sp!, {r1, lr}
+  .cfi_adjust_cfa_offset 8
+  .cfi_rel_offset lr, 4
+  mov r0, r3
+  bl __bionic_setjmp_cookie_check
+
+  // Restore return value/address.
+  ldmfd sp!, {r0, lr}
+  .cfi_adjust_cfa_offset -8
+  .cfi_restore lr
 
-  // Set return value.
-  mov r0, r1
   teq r0, #0
   moveq r0, #1
   bx lr

libc/arch-arm/bionic/vfork.S

@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <private/bionic_asm.h>
+
+ENTRY(vfork)
+    // __get_tls()[TLS_SLOT_THREAD_ID]->cached_pid_ = 0
+    mrc     p15, 0, r3, c13, c0, 3
+    ldr     r3, [r3, #4]
+    mov     r0, #0
+    str     r0, [r3, #12]
+
+    mov     ip, r7
+    ldr     r7, =__NR_vfork
+    swi     #0
+    mov     r7, ip
+    cmn     r0, #(MAX_ERRNO + 1)
+    bxls    lr
+    neg     r0, r0
+    b       __set_errno_internal
+END(vfork)

libc/arch-arm/cortex-a15/bionic/__strcat_chk.S

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013 The Android Open Source Project
+ * Copyright (C) 2015 The Android Open Source Project
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -26,191 +26,7 @@
  * SUCH DAMAGE.
  */
 
-#include <private/bionic_asm.h>
-#include <private/libc_events.h>
+// Indicate which memcpy base file to include.
+#define MEMCPY_BASE "memcpy_base.S"
 
-    .syntax unified
-
-    .thumb
-    .thumb_func
-
-// Get the length of src string, then get the source of the dst string.
-// Check that the two lengths together don't exceed the threshold, then
-// do a memcpy of the data.
-ENTRY(__strcat_chk)
-    pld     [r0, #0]
-    push    {r0, lr}
-    .cfi_def_cfa_offset 8
-    .cfi_rel_offset r0, 0
-    .cfi_rel_offset lr, 4
-    push    {r4, r5}
-    .cfi_adjust_cfa_offset 8
-    .cfi_rel_offset r4, 0
-    .cfi_rel_offset r5, 4
-
-    mov     lr, r2
-
-    // Save the dst register to r5
-    mov     r5, r0
-
-    // Zero out r4
-    eor     r4, r4, r4
-
-    // r1 contains the address of the string to count.
-.L_strlen_start:
-    mov     r0, r1
-    ands    r3, r1, #7
-    beq     .L_mainloop
-
-    // Align to a double word (64 bits).
-    rsb     r3, r3, #8
-    lsls    ip, r3, #31
-    beq     .L_align_to_32
-
-    ldrb    r2, [r1], #1
-    cbz     r2, .L_update_count_and_finish
-
-.L_align_to_32:
-    bcc     .L_align_to_64
-    ands    ip, r3, #2
-    beq     .L_align_to_64
-
-    ldrb    r2, [r1], #1
-    cbz     r2, .L_update_count_and_finish
-    ldrb    r2, [r1], #1
-    cbz     r2, .L_update_count_and_finish
-
-.L_align_to_64:
-    tst     r3, #4
-    beq     .L_mainloop
-    ldr     r3, [r1], #4
-
-    sub     ip, r3, #0x01010101
-    bic     ip, ip, r3
-    ands    ip, ip, #0x80808080
-    bne     .L_zero_in_second_register
-
-    .p2align 2
-.L_mainloop:
-    ldrd    r2, r3, [r1], #8
-
-    pld     [r1, #64]
-
-    sub     ip, r2, #0x01010101
-    bic     ip, ip, r2
-    ands    ip, ip, #0x80808080
-    bne     .L_zero_in_first_register
-
-    sub     ip, r3, #0x01010101
-    bic     ip, ip, r3
-    ands    ip, ip, #0x80808080
-    bne     .L_zero_in_second_register
-    b       .L_mainloop
-
-.L_update_count_and_finish:
-    sub     r3, r1, r0
-    sub     r3, r3, #1
-    b       .L_finish
-
-.L_zero_in_first_register:
-    sub     r3, r1, r0
-    lsls    r2, ip, #17
-    bne     .L_sub8_and_finish
-    bcs     .L_sub7_and_finish
-    lsls    ip, ip, #1
-    bne     .L_sub6_and_finish
-
-    sub     r3, r3, #5
-    b       .L_finish
-
-.L_sub8_and_finish:
-    sub     r3, r3, #8
-    b       .L_finish
-
-.L_sub7_and_finish:
-    sub     r3, r3, #7
-    b       .L_finish
-
-.L_sub6_and_finish:
-    sub     r3, r3, #6
-    b       .L_finish
-
-.L_zero_in_second_register:
-    sub     r3, r1, r0
-    lsls    r2, ip, #17
-    bne     .L_sub4_and_finish
-    bcs     .L_sub3_and_finish
-    lsls    ip, ip, #1
-    bne     .L_sub2_and_finish
-
-    sub     r3, r3, #1
-    b       .L_finish
-
-.L_sub4_and_finish:
-    sub     r3, r3, #4
-    b       .L_finish
-
-.L_sub3_and_finish:
-    sub     r3, r3, #3
-    b       .L_finish
-
-.L_sub2_and_finish:
-    sub     r3, r3, #2
-
-.L_finish:
-    cmp     r4, #0
-    bne     .L_strlen_done
-
-    // Time to get the dst string length.
-    mov     r1, r5
-
-    // Save the original source address to r5.
-    mov     r5, r0
-
-    // Save the current length (adding 1 for the terminator).
-    add     r4, r3, #1
-    b       .L_strlen_start
-
-    // r0 holds the pointer to the dst string.
-    // r3 holds the dst string length.
-    // r4 holds the src string length + 1.
-.L_strlen_done:
-    add     r2, r3, r4
-    cmp     r2, lr
-    bhi     __strcat_chk_failed
-
-    // Set up the registers for the memcpy code.
-    mov     r1, r5
-    pld     [r1, #64]
-    mov     r2, r4
-    add     r0, r0, r3
-    pop     {r4, r5}
-END(__strcat_chk)
-
-#define MEMCPY_BASE         __strcat_chk_memcpy_base
-#define MEMCPY_BASE_ALIGNED __strcat_chk_memcpy_base_aligned
-
-#include "memcpy_base.S"
-
-ENTRY_PRIVATE(__strcat_chk_failed)
-    .cfi_def_cfa_offset 8
-    .cfi_rel_offset r0, 0
-    .cfi_rel_offset lr, 4
-    .cfi_adjust_cfa_offset 8
-    .cfi_rel_offset r4, 0
-    .cfi_rel_offset r5, 4
-
-    ldr     r0, error_message
-    ldr     r1, error_code
-1:
-    add     r0, pc
-    bl      __fortify_chk_fail
-error_code:
-    .word   BIONIC_EVENT_STRCAT_BUFFER_OVERFLOW
-error_message:
-    .word   error_string-(1b+4)
-END(__strcat_chk_failed)
-
-    .data
-error_string:
-    .string "strcat: prevented write past end of buffer"
+#include "__strcat_chk_common.S"

libc/arch-arm/cortex-a15/bionic/__strcat_chk_common.S

@@ -0,0 +1,212 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <private/bionic_asm.h>
+#include <private/libc_events.h>
+
+    .syntax unified
+
+    .thumb
+    .thumb_func
+
+// Get the length of src string, then get the source of the dst string.
+// Check that the two lengths together don't exceed the threshold, then
+// do a memcpy of the data.
+ENTRY(__strcat_chk)
+    pld     [r0, #0]
+    push    {r0, lr}
+    .cfi_def_cfa_offset 8
+    .cfi_rel_offset r0, 0
+    .cfi_rel_offset lr, 4
+    push    {r4, r5}
+    .cfi_adjust_cfa_offset 8
+    .cfi_rel_offset r4, 0
+    .cfi_rel_offset r5, 4
+
+    mov     lr, r2
+
+    // Save the dst register to r5
+    mov     r5, r0
+
+    // Zero out r4
+    eor     r4, r4, r4
+
+    // r1 contains the address of the string to count.
+.L_strlen_start:
+    mov     r0, r1
+    ands    r3, r1, #7
+    beq     .L_mainloop
+
+    // Align to a double word (64 bits).
+    rsb     r3, r3, #8
+    lsls    ip, r3, #31
+    beq     .L_align_to_32
+
+    ldrb    r2, [r1], #1
+    cbz     r2, .L_update_count_and_finish
+
+.L_align_to_32:
+    bcc     .L_align_to_64
+    ands    ip, r3, #2
+    beq     .L_align_to_64
+
+    ldrb    r2, [r1], #1
+    cbz     r2, .L_update_count_and_finish
+    ldrb    r2, [r1], #1
+    cbz     r2, .L_update_count_and_finish
+
+.L_align_to_64:
+    tst     r3, #4
+    beq     .L_mainloop
+    ldr     r3, [r1], #4
+
+    sub     ip, r3, #0x01010101
+    bic     ip, ip, r3
+    ands    ip, ip, #0x80808080
+    bne     .L_zero_in_second_register
+
+    .p2align 2
+.L_mainloop:
+    ldrd    r2, r3, [r1], #8
+
+    pld     [r1, #64]
+
+    sub     ip, r2, #0x01010101
+    bic     ip, ip, r2
+    ands    ip, ip, #0x80808080
+    bne     .L_zero_in_first_register
+
+    sub     ip, r3, #0x01010101
+    bic     ip, ip, r3
+    ands    ip, ip, #0x80808080
+    bne     .L_zero_in_second_register
+    b       .L_mainloop
+
+.L_update_count_and_finish:
+    sub     r3, r1, r0
+    sub     r3, r3, #1
+    b       .L_finish
+
+.L_zero_in_first_register:
+    sub     r3, r1, r0
+    lsls    r2, ip, #17
+    bne     .L_sub8_and_finish
+    bcs     .L_sub7_and_finish
+    lsls    ip, ip, #1
+    bne     .L_sub6_and_finish
+
+    sub     r3, r3, #5
+    b       .L_finish
+
+.L_sub8_and_finish:
+    sub     r3, r3, #8
+    b       .L_finish
+
+.L_sub7_and_finish:
+    sub     r3, r3, #7
+    b       .L_finish
+
+.L_sub6_and_finish:
+    sub     r3, r3, #6
+    b       .L_finish
+
+.L_zero_in_second_register:
+    sub     r3, r1, r0
+    lsls    r2, ip, #17
+    bne     .L_sub4_and_finish
+    bcs     .L_sub3_and_finish
+    lsls    ip, ip, #1
+    bne     .L_sub2_and_finish
+
+    sub     r3, r3, #1
+    b       .L_finish
+
+.L_sub4_and_finish:
+    sub     r3, r3, #4
+    b       .L_finish
+
+.L_sub3_and_finish:
+    sub     r3, r3, #3
+    b       .L_finish
+
+.L_sub2_and_finish:
+    sub     r3, r3, #2
+
+.L_finish:
+    cmp     r4, #0
+    bne     .L_strlen_done
+
+    // Time to get the dst string length.
+    mov     r1, r5
+
+    // Save the original source address to r5.
+    mov     r5, r0
+
+    // Save the current length (adding 1 for the terminator).
+    add     r4, r3, #1
+    b       .L_strlen_start
+
+    // r0 holds the pointer to the dst string.
+    // r3 holds the dst string length.
+    // r4 holds the src string length + 1.
+.L_strlen_done:
+    add     r2, r3, r4
+    cmp     r2, lr
+    bhi     .L_strcat_chk_failed
+
+    // Set up the registers for the memcpy code.
+    mov     r1, r5
+    pld     [r1, #64]
+    mov     r2, r4
+    add     r0, r0, r3
+    pop     {r4, r5}
+    .cfi_adjust_cfa_offset -8
+    .cfi_restore r4
+    .cfi_restore r5
+
+#include MEMCPY_BASE
+
+    // Undo the above cfi directives
+    .cfi_adjust_cfa_offset 8
+    .cfi_rel_offset r4, 0
+    .cfi_rel_offset r5, 4
+.L_strcat_chk_failed:
+    ldr     r0, error_message
+    ldr     r1, error_code
+1:
+    add     r0, pc
+    bl      __fortify_chk_fail
+error_code:
+    .word   BIONIC_EVENT_STRCAT_BUFFER_OVERFLOW
+error_message:
+    .word   error_string-(1b+4)
+END(__strcat_chk)
+
+    .data
+error_string:
+    .string "strcat: prevented write past end of buffer"

libc/arch-arm/cortex-a15/bionic/__strcpy_chk.S

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013 The Android Open Source Project
+ * Copyright (C) 2015 The Android Open Source Project
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -26,155 +26,7 @@
  * SUCH DAMAGE.
  */
 
-#include <private/bionic_asm.h>
-#include <private/libc_events.h>
+// Indicate which memcpy base file to include.
+#define MEMCPY_BASE "memcpy_base.S"
 
-    .syntax unified
-
-    .thumb
-    .thumb_func
-
-// Get the length of the source string first, then do a memcpy of the data
-// instead of a strcpy.
-ENTRY(__strcpy_chk)
-    pld     [r0, #0]
-    push    {r0, lr}
-    .cfi_def_cfa_offset 8
-    .cfi_rel_offset r0, 0
-    .cfi_rel_offset lr, 4
-
-    mov     lr, r2
-    mov     r0, r1
-
-    ands    r3, r1, #7
-    beq     .L_mainloop
-
-    // Align to a double word (64 bits).
-    rsb     r3, r3, #8
-    lsls    ip, r3, #31
-    beq     .L_align_to_32
-
-    ldrb    r2, [r0], #1
-    cbz     r2, .L_update_count_and_finish
-
-.L_align_to_32:
-    bcc     .L_align_to_64
-    ands    ip, r3, #2
-    beq     .L_align_to_64
-
-    ldrb    r2, [r0], #1
-    cbz     r2, .L_update_count_and_finish
-    ldrb    r2, [r0], #1
-    cbz     r2, .L_update_count_and_finish
-
-.L_align_to_64:
-    tst     r3, #4
-    beq     .L_mainloop
-    ldr     r3, [r0], #4
-
-    sub     ip, r3, #0x01010101
-    bic     ip, ip, r3
-    ands    ip, ip, #0x80808080
-    bne     .L_zero_in_second_register
-
-    .p2align 2
-.L_mainloop:
-    ldrd    r2, r3, [r0], #8
-
-    pld     [r0, #64]
-
-    sub     ip, r2, #0x01010101
-    bic     ip, ip, r2
-    ands    ip, ip, #0x80808080
-    bne     .L_zero_in_first_register
-
-    sub     ip, r3, #0x01010101
-    bic     ip, ip, r3
-    ands    ip, ip, #0x80808080
-    bne     .L_zero_in_second_register
-    b       .L_mainloop
-
-.L_update_count_and_finish:
-    sub     r3, r0, r1
-    sub     r3, r3, #1
-    b       .L_check_size
-
-.L_zero_in_first_register:
-    sub     r3, r0, r1
-    lsls    r2, ip, #17
-    bne     .L_sub8_and_finish
-    bcs     .L_sub7_and_finish
-    lsls    ip, ip, #1
-    bne     .L_sub6_and_finish
-
-    sub     r3, r3, #5
-    b       .L_check_size
-
-.L_sub8_and_finish:
-    sub     r3, r3, #8
-    b       .L_check_size
-
-.L_sub7_and_finish:
-    sub     r3, r3, #7
-    b       .L_check_size
-
-.L_sub6_and_finish:
-    sub     r3, r3, #6
-    b       .L_check_size
-
-.L_zero_in_second_register:
-    sub     r3, r0, r1
-    lsls    r2, ip, #17
-    bne     .L_sub4_and_finish
-    bcs     .L_sub3_and_finish
-    lsls    ip, ip, #1
-    bne     .L_sub2_and_finish
-
-    sub     r3, r3, #1
-    b       .L_check_size
-
-.L_sub4_and_finish:
-    sub     r3, r3, #4
-    b       .L_check_size
-
-.L_sub3_and_finish:
-    sub     r3, r3, #3
-    b       .L_check_size
-
-.L_sub2_and_finish:
-    sub     r3, r3, #2
-
-.L_check_size:
-    pld     [r1, #0]
-    pld     [r1, #64]
-    ldr     r0, [sp]
-    cmp     r3, lr
-    bhs     __strcpy_chk_failed
-
-    // Add 1 for copy length to get the string terminator.
-    add     r2, r3, #1
-END(__strcpy_chk)
-
-#define MEMCPY_BASE         __strcpy_chk_memcpy_base
-#define MEMCPY_BASE_ALIGNED __strcpy_chk_memcpy_base_aligned
-#include "memcpy_base.S"
-
-ENTRY_PRIVATE(__strcpy_chk_failed)
-    .cfi_def_cfa_offset 8
-    .cfi_rel_offset r0, 0
-    .cfi_rel_offset lr, 4
-
-    ldr     r0, error_message
-    ldr     r1, error_code
-1:
-    add     r0, pc
-    bl      __fortify_chk_fail
-error_code:
-    .word   BIONIC_EVENT_STRCPY_BUFFER_OVERFLOW
-error_message:
-    .word   error_string-(1b+4)
-END(__strcpy_chk_failed)
-
-    .data
-error_string:
-    .string "strcpy: prevented write past end of buffer"
+#include "__strcpy_chk_common.S"

libc/arch-arm/cortex-a15/bionic/__strcpy_chk_common.S

@@ -0,0 +1,173 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <private/bionic_asm.h>
+#include <private/libc_events.h>
+
+    .syntax unified
+
+    .thumb
+    .thumb_func
+
+// Get the length of the source string first, then do a memcpy of the data
+// instead of a strcpy.
+ENTRY(__strcpy_chk)
+    pld     [r0, #0]
+    push    {r0, lr}
+    .cfi_def_cfa_offset 8
+    .cfi_rel_offset r0, 0
+    .cfi_rel_offset lr, 4
+
+    mov     lr, r2
+    mov     r0, r1
+
+    ands    r3, r1, #7
+    beq     .L_mainloop
+
+    // Align to a double word (64 bits).
+    rsb     r3, r3, #8
+    lsls    ip, r3, #31
+    beq     .L_align_to_32
+
+    ldrb    r2, [r0], #1
+    cbz     r2, .L_update_count_and_finish
+
+.L_align_to_32:
+    bcc     .L_align_to_64
+    ands    ip, r3, #2
+    beq     .L_align_to_64
+
+    ldrb    r2, [r0], #1
+    cbz     r2, .L_update_count_and_finish
+    ldrb    r2, [r0], #1
+    cbz     r2, .L_update_count_and_finish
+
+.L_align_to_64:
+    tst     r3, #4
+    beq     .L_mainloop
+    ldr     r3, [r0], #4
+
+    sub     ip, r3, #0x01010101
+    bic     ip, ip, r3
+    ands    ip, ip, #0x80808080
+    bne     .L_zero_in_second_register
+
+    .p2align 2
+.L_mainloop:
+    ldrd    r2, r3, [r0], #8
+
+    pld     [r0, #64]
+
+    sub     ip, r2, #0x01010101
+    bic     ip, ip, r2
+    ands    ip, ip, #0x80808080
+    bne     .L_zero_in_first_register
+
+    sub     ip, r3, #0x01010101
+    bic     ip, ip, r3
+    ands    ip, ip, #0x80808080
+    bne     .L_zero_in_second_register
+    b       .L_mainloop
+
+.L_update_count_and_finish:
+    sub     r3, r0, r1
+    sub     r3, r3, #1
+    b       .L_check_size
+
+.L_zero_in_first_register:
+    sub     r3, r0, r1
+    lsls    r2, ip, #17
+    bne     .L_sub8_and_finish
+    bcs     .L_sub7_and_finish
+    lsls    ip, ip, #1
+    bne     .L_sub6_and_finish
+
+    sub     r3, r3, #5
+    b       .L_check_size
+
+.L_sub8_and_finish:
+    sub     r3, r3, #8
+    b       .L_check_size
+
+.L_sub7_and_finish:
+    sub     r3, r3, #7
+    b       .L_check_size
+
+.L_sub6_and_finish:
+    sub     r3, r3, #6
+    b       .L_check_size
+
+.L_zero_in_second_register:
+    sub     r3, r0, r1
+    lsls    r2, ip, #17
+    bne     .L_sub4_and_finish
+    bcs     .L_sub3_and_finish
+    lsls    ip, ip, #1
+    bne     .L_sub2_and_finish
+
+    sub     r3, r3, #1
+    b       .L_check_size
+
+.L_sub4_and_finish:
+    sub     r3, r3, #4
+    b       .L_check_size
+
+.L_sub3_and_finish:
+    sub     r3, r3, #3
+    b       .L_check_size
+
+.L_sub2_and_finish:
+    sub     r3, r3, #2
+
+.L_check_size:
+    pld     [r1, #0]
+    pld     [r1, #64]
+    ldr     r0, [sp]
+    cmp     r3, lr
+    bhs     .L_strcpy_chk_failed
+
+    // Add 1 for copy length to get the string terminator.
+    add     r2, r3, #1
+
+#include MEMCPY_BASE
+
+.L_strcpy_chk_failed:
+    ldr     r0, error_message
+    ldr     r1, error_code
+1:
+    add     r0, pc
+    bl      __fortify_chk_fail
+error_code:
+    .word   BIONIC_EVENT_STRCPY_BUFFER_OVERFLOW
+error_message:
+    .word   error_string-(1b+4)
+END(__strcpy_chk)
+
+    .data
+error_string:
+    .string "strcpy: prevented write past end of buffer"

libc/arch-arm/cortex-a15/bionic/memcpy.S

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2008 The Android Open Source Project
+ * Copyright (C) 2015 The Android Open Source Project
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -25,79 +25,8 @@
  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
-/*
- * Copyright (c) 2013 ARM Ltd
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the company may not be used to endorse or promote
- *    products derived from this software without specific prior written
- *    permission.
- *
- * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
 
-// Prototype: void *memcpy (void *dst, const void *src, size_t count).
+// Indicate which memcpy base file to include.
+#define MEMCPY_BASE "memcpy_base.S"
 
-#include <private/bionic_asm.h>
-#include <private/libc_events.h>
-
-        .text
-        .syntax unified
-        .fpu    neon
-
-ENTRY(__memcpy_chk)
-        cmp     r2, r3
-        bhi     __memcpy_chk_fail
-
-        // Fall through to memcpy...
-END(__memcpy_chk)
-
-ENTRY(memcpy)
-        pld     [r1, #64]
-        push    {r0, lr}
-        .cfi_def_cfa_offset 8
-        .cfi_rel_offset r0, 0
-        .cfi_rel_offset lr, 4
-END(memcpy)
-
-#define MEMCPY_BASE         __memcpy_base
-#define MEMCPY_BASE_ALIGNED __memcpy_base_aligned
-#include "memcpy_base.S"
-
-ENTRY_PRIVATE(__memcpy_chk_fail)
-        // Preserve lr for backtrace.
-        push    {lr}
-        .cfi_def_cfa_offset 4
-        .cfi_rel_offset lr, 0
-
-        ldr     r0, error_message
-        ldr     r1, error_code
-1:
-        add     r0, pc
-        bl      __fortify_chk_fail
-error_code:
-        .word   BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW
-error_message:
-        .word   error_string-(1b+8)
-END(__memcpy_chk_fail)
-
-        .data
-error_string:
-        .string "memcpy: prevented write past end of buffer"
+#include "memcpy_common.S"

libc/arch-arm/cortex-a15/bionic/memcpy_base.S

@@ -53,11 +53,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-ENTRY_PRIVATE(MEMCPY_BASE)
-        .cfi_def_cfa_offset 8
-        .cfi_rel_offset r0, 0
-        .cfi_rel_offset lr, 4
-
+.L_memcpy_base:
         // Assumes that n >= 0, and dst, src are valid pointers.
         // For any sizes less than 832 use the neon code that doesn't
         // care about the src alignment. This avoids any checks
@@ -168,12 +164,6 @@ ENTRY_PRIVATE(MEMCPY_BASE)
         eor     r3, r0, r1
         ands    r3, r3, #0x3
         bne     .L_copy_unknown_alignment
-END(MEMCPY_BASE)
-
-ENTRY_PRIVATE(MEMCPY_BASE_ALIGNED)
-        .cfi_def_cfa_offset 8
-        .cfi_rel_offset r0, 0
-        .cfi_rel_offset lr, 4
 
         // To try and improve performance, stack layout changed,
         // i.e., not keeping the stack looking like users expect
@@ -185,7 +175,7 @@ ENTRY_PRIVATE(MEMCPY_BASE_ALIGNED)
         strd    r6, r7, [sp, #-8]!
         .cfi_adjust_cfa_offset 8
         .cfi_rel_offset r6, 0
-        .cfi_rel_offset r7, 0
+        .cfi_rel_offset r7, 4
         strd    r8, r9, [sp, #-8]!
         .cfi_adjust_cfa_offset 8
         .cfi_rel_offset r8, 0
@@ -291,10 +281,28 @@ ENTRY_PRIVATE(MEMCPY_BASE_ALIGNED)
 
         // Restore registers: optimized pop {r0, pc}
         ldrd    r8, r9, [sp], #8
+        .cfi_adjust_cfa_offset -8
+        .cfi_restore r8
+        .cfi_restore r9
         ldrd    r6, r7, [sp], #8
+        .cfi_adjust_cfa_offset -8
+        .cfi_restore r6
+        .cfi_restore r7
         ldrd    r4, r5, [sp], #8
+        .cfi_adjust_cfa_offset -8
+        .cfi_restore r4
+        .cfi_restore r5
         pop     {r0, pc}
 
+        // Put the cfi directives back for the below instructions.
+        .cfi_adjust_cfa_offset 24
+        .cfi_rel_offset r4, 0
+        .cfi_rel_offset r5, 4
+        .cfi_rel_offset r6, 8
+        .cfi_rel_offset r7, 12
+        .cfi_rel_offset r8, 16
+        .cfi_rel_offset r9, 20
+
 .L_dst_not_word_aligned:
         // Align dst to word.
         rsb     ip, ip, #4
@@ -315,4 +323,12 @@ ENTRY_PRIVATE(MEMCPY_BASE_ALIGNED)
 
         // Src is guaranteed to be at least word aligned by this point.
         b       .L_word_aligned
-END(MEMCPY_BASE_ALIGNED)
+
+        // Undo any cfi directives from above.
+        .cfi_adjust_cfa_offset -24
+        .cfi_restore r4
+        .cfi_restore r5
+        .cfi_restore r6
+        .cfi_restore r7
+        .cfi_restore r8
+        .cfi_restore r9

libc/arch-arm/cortex-a15/bionic/memcpy_common.S

@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2008 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+/*
+ * Copyright (c) 2013 ARM Ltd
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the company may not be used to endorse or promote
+ *    products derived from this software without specific prior written
+ *    permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <private/bionic_asm.h>
+#include <private/libc_events.h>
+
+        .text
+        .syntax unified
+        .fpu    neon
+
+ENTRY(__memcpy_chk)
+        cmp     r2, r3
+        bhi     .L_memcpy_chk_fail
+
+        // Fall through to memcpy...
+END(__memcpy_chk)
+
+// Prototype: void *memcpy (void *dst, const void *src, size_t count).
+ENTRY(memcpy)
+        pld     [r1, #64]
+        push    {r0, lr}
+        .cfi_def_cfa_offset 8
+        .cfi_rel_offset r0, 0
+        .cfi_rel_offset lr, 4
+
+#include MEMCPY_BASE
+
+        // Undo the cfi instructions from above.
+        .cfi_def_cfa_offset 0
+        .cfi_restore r0
+        .cfi_restore lr
+.L_memcpy_chk_fail:
+        // Preserve lr for backtrace.
+        push    {lr}
+        .cfi_adjust_cfa_offset 4
+        .cfi_rel_offset lr, 0
+
+        ldr     r0, error_message
+        ldr     r1, error_code
+1:
+        add     r0, pc
+        bl      __fortify_chk_fail
+error_code:
+        .word   BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW
+error_message:
+        .word   error_string-(1b+8)
+END(memcpy)
+
+        .data
+error_string:
+        .string "memcpy: prevented write past end of buffer"

libc/arch-arm/cortex-a15/bionic/strcat.S

@@ -70,7 +70,7 @@
 
     .macro m_scan_byte
     ldrb    r3, [r0]
-    cbz     r3, strcat_r0_scan_done
+    cbz     r3, .L_strcat_r0_scan_done
     add     r0, #1
     .endm // m_scan_byte
 
@@ -84,10 +84,10 @@ ENTRY(strcat)
     // Quick check to see if src is empty.
     ldrb    r2, [r1]
     pld     [r1, #0]
-    cbnz    r2, strcat_continue
+    cbnz    r2, .L_strcat_continue
     bx      lr
 
-strcat_continue:
+.L_strcat_continue:
     // To speed up really small dst strings, unroll checking the first 4 bytes.
     m_push
     m_scan_byte
@@ -96,95 +96,102 @@ strcat_continue:
     m_scan_byte
 
     ands    r3, r0, #7
-    beq     strcat_mainloop
+    beq     .L_strcat_mainloop
 
     // Align to a double word (64 bits).
     rsb     r3, r3, #8
     lsls    ip, r3, #31
-    beq     strcat_align_to_32
+    beq     .L_strcat_align_to_32
 
     ldrb    r5, [r0]
-    cbz     r5, strcat_r0_scan_done
+    cbz     r5, .L_strcat_r0_scan_done
     add     r0, r0, #1
 
-strcat_align_to_32:
-    bcc     strcat_align_to_64
+.L_strcat_align_to_32:
+    bcc     .L_strcat_align_to_64
 
     ldrb    r2, [r0]
-    cbz     r2, strcat_r0_scan_done
+    cbz     r2, .L_strcat_r0_scan_done
     add     r0, r0, #1
     ldrb    r4, [r0]
-    cbz     r4, strcat_r0_scan_done
+    cbz     r4, .L_strcat_r0_scan_done
     add     r0, r0, #1
 
-strcat_align_to_64:
+.L_strcat_align_to_64:
     tst     r3, #4
-    beq     strcat_mainloop
+    beq     .L_strcat_mainloop
     ldr     r3, [r0], #4
 
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     strcat_zero_in_second_register
-    b       strcat_mainloop
+    bne     .L_strcat_zero_in_second_register
+    b       .L_strcat_mainloop
 
-strcat_r0_scan_done:
+.L_strcat_r0_scan_done:
     // For short copies, hard-code checking the first 8 bytes since this
     // new code doesn't win until after about 8 bytes.
-    m_copy_byte reg=r2, cmd=cbz, label=strcpy_finish
-    m_copy_byte reg=r3, cmd=cbz, label=strcpy_finish
-    m_copy_byte reg=r4, cmd=cbz, label=strcpy_finish
-    m_copy_byte reg=r5, cmd=cbz, label=strcpy_finish
-    m_copy_byte reg=r2, cmd=cbz, label=strcpy_finish
-    m_copy_byte reg=r3, cmd=cbz, label=strcpy_finish
-    m_copy_byte reg=r4, cmd=cbz, label=strcpy_finish
-    m_copy_byte reg=r5, cmd=cbnz, label=strcpy_continue
+    m_copy_byte reg=r2, cmd=cbz, label=.L_strcpy_finish
+    m_copy_byte reg=r3, cmd=cbz, label=.L_strcpy_finish
+    m_copy_byte reg=r4, cmd=cbz, label=.L_strcpy_finish
+    m_copy_byte reg=r5, cmd=cbz, label=.L_strcpy_finish
+    m_copy_byte reg=r2, cmd=cbz, label=.L_strcpy_finish
+    m_copy_byte reg=r3, cmd=cbz, label=.L_strcpy_finish
+    m_copy_byte reg=r4, cmd=cbz, label=.L_strcpy_finish
+    m_copy_byte reg=r5, cmd=cbnz, label=.L_strcpy_continue
 
-strcpy_finish:
+.L_strcpy_finish:
     m_pop
 
-strcpy_continue:
+.L_strcpy_continue:
     ands    r3, r0, #7
-    beq     strcpy_check_src_align
+    beq     .L_strcpy_check_src_align
 
     // Align to a double word (64 bits).
     rsb     r3, r3, #8
     lsls    ip, r3, #31
-    beq     strcpy_align_to_32
+    beq     .L_strcpy_align_to_32
 
     ldrb    r2, [r1], #1
     strb    r2, [r0], #1
-    cbz     r2, strcpy_complete
+    cbz     r2, .L_strcpy_complete
 
-strcpy_align_to_32:
-    bcc     strcpy_align_to_64
+.L_strcpy_align_to_32:
+    bcc     .L_strcpy_align_to_64
 
     ldrb    r2, [r1], #1
     strb    r2, [r0], #1
-    cbz     r2, strcpy_complete
+    cbz     r2, .L_strcpy_complete
     ldrb    r2, [r1], #1
     strb    r2, [r0], #1
-    cbz     r2, strcpy_complete
+    cbz     r2, .L_strcpy_complete
 
-strcpy_align_to_64:
+.L_strcpy_align_to_64:
     tst     r3, #4
-    beq     strcpy_check_src_align
-    ldr     r2, [r1], #4
+    beq     .L_strcpy_check_src_align
+    // Read one byte at a time since we don't know the src alignment
+    // and we don't want to read into a different page.
+    ldrb    r2, [r1], #1
+    strb    r2, [r0], #1
+    cbz     r2, .L_strcpy_complete
+    ldrb    r2, [r1], #1
+    strb    r2, [r0], #1
+    cbz     r2, .L_strcpy_complete
+    ldrb    r2, [r1], #1
+    strb    r2, [r0], #1
+    cbz     r2, .L_strcpy_complete
+    ldrb    r2, [r1], #1
+    strb    r2, [r0], #1
+    cbz     r2, .L_strcpy_complete
 
-    sub     ip, r2, #0x01010101
-    bic     ip, ip, r2
-    ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_first_register
-    str     r2, [r0], #4
-
-strcpy_check_src_align:
+.L_strcpy_check_src_align:
     // At this point dst is aligned to a double word, check if src
     // is also aligned to a double word.
     ands    r3, r1, #7
-    bne     strcpy_unaligned_copy
+    bne     .L_strcpy_unaligned_copy
 
     .p2align 2
-strcpy_mainloop:
+.L_strcpy_mainloop:
     ldrd    r2, r3, [r1], #8
 
     pld     [r1, #64]
@@ -192,128 +199,128 @@ strcpy_mainloop:
     sub     ip, r2, #0x01010101
     bic     ip, ip, r2
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_first_register
+    bne     .L_strcpy_zero_in_first_register
 
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_second_register
+    bne     .L_strcpy_zero_in_second_register
 
     strd    r2, r3, [r0], #8
-    b       strcpy_mainloop
+    b       .L_strcpy_mainloop
 
-strcpy_complete:
+.L_strcpy_complete:
     m_pop
 
-strcpy_zero_in_first_register:
+.L_strcpy_zero_in_first_register:
     lsls    lr, ip, #17
-    bne     strcpy_copy1byte
-    bcs     strcpy_copy2bytes
+    bne     .L_strcpy_copy1byte
+    bcs     .L_strcpy_copy2bytes
     lsls    ip, ip, #1
-    bne     strcpy_copy3bytes
+    bne     .L_strcpy_copy3bytes
 
-strcpy_copy4bytes:
+.L_strcpy_copy4bytes:
     // Copy 4 bytes to the destiniation.
     str     r2, [r0]
     m_pop
 
-strcpy_copy1byte:
+.L_strcpy_copy1byte:
     strb    r2, [r0]
     m_pop
 
-strcpy_copy2bytes:
+.L_strcpy_copy2bytes:
     strh    r2, [r0]
     m_pop
 
-strcpy_copy3bytes:
+.L_strcpy_copy3bytes:
     strh    r2, [r0], #2
     lsr     r2, #16
     strb    r2, [r0]
     m_pop
 
-strcpy_zero_in_second_register:
+.L_strcpy_zero_in_second_register:
     lsls    lr, ip, #17
-    bne     strcpy_copy5bytes
-    bcs     strcpy_copy6bytes
+    bne     .L_strcpy_copy5bytes
+    bcs     .L_strcpy_copy6bytes
     lsls    ip, ip, #1
-    bne     strcpy_copy7bytes
+    bne     .L_strcpy_copy7bytes
 
     // Copy 8 bytes to the destination.
     strd    r2, r3, [r0]
     m_pop
 
-strcpy_copy5bytes:
+.L_strcpy_copy5bytes:
     str     r2, [r0], #4
     strb    r3, [r0]
     m_pop
 
-strcpy_copy6bytes:
+.L_strcpy_copy6bytes:
     str     r2, [r0], #4
     strh    r3, [r0]
     m_pop
 
-strcpy_copy7bytes:
+.L_strcpy_copy7bytes:
     str     r2, [r0], #4
     strh    r3, [r0], #2
     lsr     r3, #16
     strb    r3, [r0]
     m_pop
 
-strcpy_unaligned_copy:
+.L_strcpy_unaligned_copy:
     // Dst is aligned to a double word, while src is at an unknown alignment.
     // There are 7 different versions of the unaligned copy code
     // to prevent overreading the src. The mainloop of every single version
     // will store 64 bits per loop. The difference is how much of src can
     // be read without potentially crossing a page boundary.
     tbb     [pc, r3]
-strcpy_unaligned_branchtable:
+.L_strcpy_unaligned_branchtable:
     .byte 0
-    .byte ((strcpy_unalign7 - strcpy_unaligned_branchtable)/2)
-    .byte ((strcpy_unalign6 - strcpy_unaligned_branchtable)/2)
-    .byte ((strcpy_unalign5 - strcpy_unaligned_branchtable)/2)
-    .byte ((strcpy_unalign4 - strcpy_unaligned_branchtable)/2)
-    .byte ((strcpy_unalign3 - strcpy_unaligned_branchtable)/2)
-    .byte ((strcpy_unalign2 - strcpy_unaligned_branchtable)/2)
-    .byte ((strcpy_unalign1 - strcpy_unaligned_branchtable)/2)
+    .byte ((.L_strcpy_unalign7 - .L_strcpy_unaligned_branchtable)/2)
+    .byte ((.L_strcpy_unalign6 - .L_strcpy_unaligned_branchtable)/2)
+    .byte ((.L_strcpy_unalign5 - .L_strcpy_unaligned_branchtable)/2)
+    .byte ((.L_strcpy_unalign4 - .L_strcpy_unaligned_branchtable)/2)
+    .byte ((.L_strcpy_unalign3 - .L_strcpy_unaligned_branchtable)/2)
+    .byte ((.L_strcpy_unalign2 - .L_strcpy_unaligned_branchtable)/2)
+    .byte ((.L_strcpy_unalign1 - .L_strcpy_unaligned_branchtable)/2)
 
     .p2align 2
     // Can read 7 bytes before possibly crossing a page.
-strcpy_unalign7:
+.L_strcpy_unalign7:
     ldr     r2, [r1], #4
 
     sub     ip, r2, #0x01010101
     bic     ip, ip, r2
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_first_register
+    bne     .L_strcpy_zero_in_first_register
 
     ldrb    r3, [r1]
-    cbz     r3, strcpy_unalign7_copy5bytes
+    cbz     r3, .L_strcpy_unalign7_copy5bytes
     ldrb    r4, [r1, #1]
-    cbz     r4, strcpy_unalign7_copy6bytes
+    cbz     r4, .L_strcpy_unalign7_copy6bytes
     ldrb    r5, [r1, #2]
-    cbz     r5, strcpy_unalign7_copy7bytes
+    cbz     r5, .L_strcpy_unalign7_copy7bytes
 
     ldr     r3, [r1], #4
     pld     [r1, #64]
 
     lsrs    ip, r3, #24
     strd    r2, r3, [r0], #8
-    beq     strcpy_unalign_return
-    b       strcpy_unalign7
+    beq     .L_strcpy_unalign_return
+    b       .L_strcpy_unalign7
 
-strcpy_unalign7_copy5bytes:
+.L_strcpy_unalign7_copy5bytes:
     str     r2, [r0], #4
     strb    r3, [r0]
-strcpy_unalign_return:
+.L_strcpy_unalign_return:
     m_pop
 
-strcpy_unalign7_copy6bytes:
+.L_strcpy_unalign7_copy6bytes:
     str     r2, [r0], #4
     strb    r3, [r0], #1
     strb    r4, [r0], #1
     m_pop
 
-strcpy_unalign7_copy7bytes:
+.L_strcpy_unalign7_copy7bytes:
     str     r2, [r0], #4
     strb    r3, [r0], #1
     strb    r4, [r0], #1
@@ -322,41 +329,41 @@ strcpy_unalign7_copy7bytes:
 
     .p2align 2
     // Can read 6 bytes before possibly crossing a page.
-strcpy_unalign6:
+.L_strcpy_unalign6:
     ldr     r2, [r1], #4
 
     sub     ip, r2, #0x01010101
     bic     ip, ip, r2
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_first_register
+    bne     .L_strcpy_zero_in_first_register
 
     ldrb    r4, [r1]
-    cbz     r4, strcpy_unalign_copy5bytes
+    cbz     r4, .L_strcpy_unalign_copy5bytes
     ldrb    r5, [r1, #1]
-    cbz     r5, strcpy_unalign_copy6bytes
+    cbz     r5, .L_strcpy_unalign_copy6bytes
 
     ldr     r3, [r1], #4
     pld     [r1, #64]
 
     tst     r3, #0xff0000
-    beq     strcpy_copy7bytes
+    beq     .L_strcpy_copy7bytes
     lsrs    ip, r3, #24
     strd    r2, r3, [r0], #8
-    beq     strcpy_unalign_return
-    b       strcpy_unalign6
+    beq     .L_strcpy_unalign_return
+    b       .L_strcpy_unalign6
 
     .p2align 2
     // Can read 5 bytes before possibly crossing a page.
-strcpy_unalign5:
+.L_strcpy_unalign5:
     ldr     r2, [r1], #4
 
     sub     ip, r2, #0x01010101
     bic     ip, ip, r2
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_first_register
+    bne     .L_strcpy_zero_in_first_register
 
     ldrb    r4, [r1]
-    cbz     r4, strcpy_unalign_copy5bytes
+    cbz     r4, .L_strcpy_unalign_copy5bytes
 
     ldr     r3, [r1], #4
 
@@ -365,17 +372,17 @@ strcpy_unalign5:
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_second_register
+    bne     .L_strcpy_zero_in_second_register
 
     strd    r2, r3, [r0], #8
-    b       strcpy_unalign5
+    b       .L_strcpy_unalign5
 
-strcpy_unalign_copy5bytes:
+.L_strcpy_unalign_copy5bytes:
     str     r2, [r0], #4
     strb    r4, [r0]
     m_pop
 
-strcpy_unalign_copy6bytes:
+.L_strcpy_unalign_copy6bytes:
     str     r2, [r0], #4
     strb    r4, [r0], #1
     strb    r5, [r0]
@@ -383,13 +390,13 @@ strcpy_unalign_copy6bytes:
 
     .p2align 2
     // Can read 4 bytes before possibly crossing a page.
-strcpy_unalign4:
+.L_strcpy_unalign4:
     ldr     r2, [r1], #4
 
     sub     ip, r2, #0x01010101
     bic     ip, ip, r2
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_first_register
+    bne     .L_strcpy_zero_in_first_register
 
     ldr     r3, [r1], #4
     pld     [r1, #64]
@@ -397,20 +404,20 @@ strcpy_unalign4:
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_second_register
+    bne     .L_strcpy_zero_in_second_register
 
     strd    r2, r3, [r0], #8
-    b       strcpy_unalign4
+    b       .L_strcpy_unalign4
 
     .p2align 2
     // Can read 3 bytes before possibly crossing a page.
-strcpy_unalign3:
+.L_strcpy_unalign3:
     ldrb    r2, [r1]
-    cbz     r2, strcpy_unalign3_copy1byte
+    cbz     r2, .L_strcpy_unalign3_copy1byte
     ldrb    r3, [r1, #1]
-    cbz     r3, strcpy_unalign3_copy2bytes
+    cbz     r3, .L_strcpy_unalign3_copy2bytes
     ldrb    r4, [r1, #2]
-    cbz     r4, strcpy_unalign3_copy3bytes
+    cbz     r4, .L_strcpy_unalign3_copy3bytes
 
     ldr     r2, [r1], #4
     ldr     r3, [r1], #4
@@ -418,26 +425,26 @@ strcpy_unalign3:
     pld     [r1, #64]
 
     lsrs    lr, r2, #24
-    beq     strcpy_copy4bytes
+    beq     .L_strcpy_copy4bytes
 
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_second_register
+    bne     .L_strcpy_zero_in_second_register
 
     strd    r2, r3, [r0], #8
-    b       strcpy_unalign3
+    b       .L_strcpy_unalign3
 
-strcpy_unalign3_copy1byte:
+.L_strcpy_unalign3_copy1byte:
     strb    r2, [r0]
     m_pop
 
-strcpy_unalign3_copy2bytes:
+.L_strcpy_unalign3_copy2bytes:
     strb    r2, [r0], #1
     strb    r3, [r0]
     m_pop
 
-strcpy_unalign3_copy3bytes:
+.L_strcpy_unalign3_copy3bytes:
     strb    r2, [r0], #1
     strb    r3, [r0], #1
     strb    r4, [r0]
@@ -445,34 +452,34 @@ strcpy_unalign3_copy3bytes:
 
     .p2align 2
     // Can read 2 bytes before possibly crossing a page.
-strcpy_unalign2:
+.L_strcpy_unalign2:
     ldrb    r2, [r1]
-    cbz     r2, strcpy_unalign_copy1byte
+    cbz     r2, .L_strcpy_unalign_copy1byte
     ldrb    r4, [r1, #1]
-    cbz     r4, strcpy_unalign_copy2bytes
+    cbz     r4, .L_strcpy_unalign_copy2bytes
 
     ldr     r2, [r1], #4
     ldr     r3, [r1], #4
     pld     [r1, #64]
 
     tst     r2, #0xff0000
-    beq     strcpy_copy3bytes
+    beq     .L_strcpy_copy3bytes
     lsrs    ip, r2, #24
-    beq     strcpy_copy4bytes
+    beq     .L_strcpy_copy4bytes
 
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_second_register
+    bne     .L_strcpy_zero_in_second_register
 
     strd    r2, r3, [r0], #8
-    b       strcpy_unalign2
+    b       .L_strcpy_unalign2
 
     .p2align 2
     // Can read 1 byte before possibly crossing a page.
-strcpy_unalign1:
+.L_strcpy_unalign1:
     ldrb    r2, [r1]
-    cbz     r2, strcpy_unalign_copy1byte
+    cbz     r2, .L_strcpy_unalign_copy1byte
 
     ldr     r2, [r1], #4
     ldr     r3, [r1], #4
@@ -482,27 +489,27 @@ strcpy_unalign1:
     sub     ip, r2, #0x01010101
     bic     ip, ip, r2
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_first_register
+    bne     .L_strcpy_zero_in_first_register
 
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_second_register
+    bne     .L_strcpy_zero_in_second_register
 
     strd    r2, r3, [r0], #8
-    b       strcpy_unalign1
+    b       .L_strcpy_unalign1
 
-strcpy_unalign_copy1byte:
+.L_strcpy_unalign_copy1byte:
     strb    r2, [r0]
     m_pop
 
-strcpy_unalign_copy2bytes:
+.L_strcpy_unalign_copy2bytes:
     strb    r2, [r0], #1
     strb    r4, [r0]
     m_pop
 
     .p2align 2
-strcat_mainloop:
+.L_strcat_mainloop:
     ldrd    r2, r3, [r0], #8
 
     pld     [r0, #64]
@@ -510,59 +517,59 @@ strcat_mainloop:
     sub     ip, r2, #0x01010101
     bic     ip, ip, r2
     ands    ip, ip, #0x80808080
-    bne     strcat_zero_in_first_register
+    bne     .L_strcat_zero_in_first_register
 
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     strcat_zero_in_second_register
-    b       strcat_mainloop
+    bne     .L_strcat_zero_in_second_register
+    b       .L_strcat_mainloop
 
-strcat_zero_in_first_register:
+.L_strcat_zero_in_first_register:
     // Prefetch the src now, it's going to be used soon.
     pld     [r1, #0]
     lsls    lr, ip, #17
-    bne     strcat_sub8
-    bcs     strcat_sub7
+    bne     .L_strcat_sub8
+    bcs     .L_strcat_sub7
     lsls    ip, ip, #1
-    bne     strcat_sub6
+    bne     .L_strcat_sub6
 
     sub     r0, r0, #5
-    b       strcat_r0_scan_done
+    b       .L_strcat_r0_scan_done
 
-strcat_sub8:
+.L_strcat_sub8:
     sub     r0, r0, #8
-    b       strcat_r0_scan_done
+    b       .L_strcat_r0_scan_done
 
-strcat_sub7:
+.L_strcat_sub7:
     sub     r0, r0, #7
-    b       strcat_r0_scan_done
+    b       .L_strcat_r0_scan_done
 
-strcat_sub6:
+.L_strcat_sub6:
     sub     r0, r0, #6
-    b       strcat_r0_scan_done
+    b       .L_strcat_r0_scan_done
 
-strcat_zero_in_second_register:
+.L_strcat_zero_in_second_register:
     // Prefetch the src now, it's going to be used soon.
     pld     [r1, #0]
     lsls    lr, ip, #17
-    bne     strcat_sub4
-    bcs     strcat_sub3
+    bne     .L_strcat_sub4
+    bcs     .L_strcat_sub3
     lsls    ip, ip, #1
-    bne     strcat_sub2
+    bne     .L_strcat_sub2
 
     sub     r0, r0, #1
-    b       strcat_r0_scan_done
+    b       .L_strcat_r0_scan_done
 
-strcat_sub4:
+.L_strcat_sub4:
     sub     r0, r0, #4
-    b       strcat_r0_scan_done
+    b       .L_strcat_r0_scan_done
 
-strcat_sub3:
+.L_strcat_sub3:
     sub     r0, r0, #3
-    b       strcat_r0_scan_done
+    b       .L_strcat_r0_scan_done
 
-strcat_sub2:
+.L_strcat_sub2:
     sub     r0, r0, #2
-    b       strcat_r0_scan_done
+    b       .L_strcat_r0_scan_done
 END(strcat)

libc/arch-arm/cortex-a15/bionic/string_copy.S

@@ -149,13 +149,20 @@ ENTRY(strcpy)
 .Lstringcopy_align_to_64:
     tst     r3, #4
     beq     .Lstringcopy_check_src_align
-    ldr     r2, [r1], #4
-
-    sub     ip, r2, #0x01010101
-    bic     ip, ip, r2
-    ands    ip, ip, #0x80808080
-    bne     .Lstringcopy_zero_in_first_register
-    str     r2, [r0], #4
+    // Read one byte at a time since we don't have any idea about the alignment
+    // of the source and we don't want to read into a different page.
+    ldrb    r2, [r1], #1
+    strb    r2, [r0], #1
+    cbz     r2, .Lstringcopy_complete
+    ldrb    r2, [r1], #1
+    strb    r2, [r0], #1
+    cbz     r2, .Lstringcopy_complete
+    ldrb    r2, [r1], #1
+    strb    r2, [r0], #1
+    cbz     r2, .Lstringcopy_complete
+    ldrb    r2, [r1], #1
+    strb    r2, [r0], #1
+    cbz     r2, .Lstringcopy_complete
 
 .Lstringcopy_check_src_align:
     // At this point dst is aligned to a double word, check if src

libc/arch-arm/cortex-a15/bionic/strlen.S

@@ -65,38 +65,38 @@ ENTRY(strlen)
     mov     r1, r0
 
     ands    r3, r0, #7
-    beq     mainloop
+    beq     .L_mainloop
 
     // Align to a double word (64 bits).
     rsb     r3, r3, #8
     lsls    ip, r3, #31
-    beq     align_to_32
+    beq     .L_align_to_32
 
     ldrb    r2, [r1], #1
-    cbz     r2, update_count_and_return
+    cbz     r2, .L_update_count_and_return
 
-align_to_32:
-    bcc     align_to_64
+.L_align_to_32:
+    bcc     .L_align_to_64
     ands    ip, r3, #2
-    beq     align_to_64
+    beq     .L_align_to_64
 
     ldrb    r2, [r1], #1
-    cbz     r2, update_count_and_return
+    cbz     r2, .L_update_count_and_return
     ldrb    r2, [r1], #1
-    cbz     r2, update_count_and_return
+    cbz     r2, .L_update_count_and_return
 
-align_to_64:
+.L_align_to_64:
     tst     r3, #4
-    beq     mainloop
+    beq     .L_mainloop
     ldr     r3, [r1], #4
 
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     zero_in_second_register
+    bne     .L_zero_in_second_register
 
     .p2align 2
-mainloop:
+.L_mainloop:
     ldrd    r2, r3, [r1], #8
 
     pld     [r1, #64]
@@ -104,62 +104,62 @@ mainloop:
     sub     ip, r2, #0x01010101
     bic     ip, ip, r2
     ands    ip, ip, #0x80808080
-    bne     zero_in_first_register
+    bne     .L_zero_in_first_register
 
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     zero_in_second_register
-    b       mainloop
+    bne     .L_zero_in_second_register
+    b       .L_mainloop
 
-update_count_and_return:
+.L_update_count_and_return:
     sub     r0, r1, r0
     sub     r0, r0, #1
     bx      lr
 
-zero_in_first_register:
+.L_zero_in_first_register:
     sub     r0, r1, r0
     lsls    r3, ip, #17
-    bne     sub8_and_return
-    bcs     sub7_and_return
+    bne     .L_sub8_and_return
+    bcs     .L_sub7_and_return
     lsls    ip, ip, #1
-    bne     sub6_and_return
+    bne     .L_sub6_and_return
 
     sub     r0, r0, #5
     bx      lr
 
-sub8_and_return:
+.L_sub8_and_return:
     sub     r0, r0, #8
     bx      lr
 
-sub7_and_return:
+.L_sub7_and_return:
     sub     r0, r0, #7
     bx      lr
 
-sub6_and_return:
+.L_sub6_and_return:
     sub     r0, r0, #6
     bx      lr
 
-zero_in_second_register:
+.L_zero_in_second_register:
     sub     r0, r1, r0
     lsls    r3, ip, #17
-    bne     sub4_and_return
-    bcs     sub3_and_return
+    bne     .L_sub4_and_return
+    bcs     .L_sub3_and_return
     lsls    ip, ip, #1
-    bne     sub2_and_return
+    bne     .L_sub2_and_return
 
     sub     r0, r0, #1
     bx      lr
 
-sub4_and_return:
+.L_sub4_and_return:
     sub     r0, r0, #4
     bx      lr
 
-sub3_and_return:
+.L_sub3_and_return:
     sub     r0, r0, #3
     bx      lr
 
-sub2_and_return:
+.L_sub2_and_return:
     sub     r0, r0, #2
     bx      lr
 END(strlen)

libc/arch-arm/cortex-a15/cortex-a15.mk

@@ -1,3 +1,17 @@
+libc_openbsd_src_files_exclude_arm += \
+    upstream-openbsd/lib/libc/string/memmove.c \
+    upstream-openbsd/lib/libc/string/stpcpy.c \
+    upstream-openbsd/lib/libc/string/strcat.c \
+
+libc_bionic_src_files_exclude_arm += \
+    arch-arm/generic/bionic/memcpy.S \
+    arch-arm/generic/bionic/memset.S \
+    arch-arm/generic/bionic/strcmp.S \
+    arch-arm/generic/bionic/strcpy.S \
+    arch-arm/generic/bionic/strlen.c \
+    bionic/__strcat_chk.cpp \
+    bionic/__strcpy_chk.cpp \
+
 libc_bionic_src_files_arm += \
     arch-arm/cortex-a15/bionic/memcpy.S \
     arch-arm/cortex-a15/bionic/memset.S \
@@ -9,8 +23,5 @@ libc_bionic_src_files_arm += \
     arch-arm/cortex-a15/bionic/__strcpy_chk.S \
     arch-arm/cortex-a15/bionic/strlen.S \
 
-libc_bionic_src_files_arm += \
-    arch-arm/generic/bionic/memcmp.S \
-
 libc_bionic_src_files_arm += \
     arch-arm/denver/bionic/memmove.S \

libc/arch-arm/cortex-a53.a57/cortex-a53.a57.mk

@@ -0,0 +1,32 @@
+# This file represents the best optimized routines that are the middle
+# ground when running on a big/little system that is cortex-a57/cortex-a53.
+# The cortex-a7 optimized routines, and the cortex-a53 optimized routines
+# decrease performance on cortex-a57 processors by as much as 20%.
+
+libc_openbsd_src_files_exclude_arm += \
+    upstream-openbsd/lib/libc/string/memmove.c \
+    upstream-openbsd/lib/libc/string/stpcpy.c \
+    upstream-openbsd/lib/libc/string/strcat.c \
+
+libc_bionic_src_files_exclude_arm += \
+    arch-arm/generic/bionic/memcpy.S \
+    arch-arm/generic/bionic/memset.S \
+    arch-arm/generic/bionic/strcmp.S \
+    arch-arm/generic/bionic/strcpy.S \
+    arch-arm/generic/bionic/strlen.c \
+    bionic/__strcat_chk.cpp \
+    bionic/__strcpy_chk.cpp \
+
+libc_bionic_src_files_arm += \
+    arch-arm/cortex-a15/bionic/memcpy.S \
+    arch-arm/cortex-a15/bionic/memset.S \
+    arch-arm/cortex-a15/bionic/stpcpy.S \
+    arch-arm/cortex-a15/bionic/strcat.S \
+    arch-arm/cortex-a15/bionic/__strcat_chk.S \
+    arch-arm/cortex-a15/bionic/strcmp.S \
+    arch-arm/cortex-a15/bionic/strcpy.S \
+    arch-arm/cortex-a15/bionic/__strcpy_chk.S \
+    arch-arm/cortex-a15/bionic/strlen.S \
+
+libc_bionic_src_files_arm += \
+    arch-arm/denver/bionic/memmove.S \

libc/arch-arm/cortex-a53/bionic/__strcat_chk.S

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+// Indicate which memcpy base file to include.
+#define MEMCPY_BASE "arch-arm/cortex-a53/bionic/memcpy_base.S"
+
+#include "arch-arm/cortex-a15/bionic/__strcat_chk_common.S"

libc/arch-arm/cortex-a53/bionic/__strcpy_chk.S

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+// Indicate which memcpy base file to include.
+#define MEMCPY_BASE "arch-arm/cortex-a53/bionic/memcpy_base.S"
+
+#include "arch-arm/cortex-a15/bionic/__strcpy_chk_common.S"

libc/arch-arm/cortex-a53/bionic/memcpy.S

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2014 The Android Open Source Project
+ * Copyright (C) 2015 The Android Open Source Project
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -26,9 +26,7 @@
  * SUCH DAMAGE.
  */
 
-#ifndef _SYS_SHM_H_
-#define _SYS_SHM_H_
+// Indicate which memcpy base file to include.
+#define MEMCPY_BASE "arch-arm/cortex-a53/bionic/memcpy_base.S"
 
-#include <linux/shm.h>
-
-#endif /* _SYS_SHM_H_ */
+#include "arch-arm/cortex-a15/bionic/memcpy_common.S"

libc/arch-arm/cortex-a53/bionic/memcpy_base.S

@@ -0,0 +1,143 @@
+/*
+ * Copyright (C) 2008 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+/*
+ * Copyright (c) 2013 ARM Ltd
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the company may not be used to endorse or promote
+ *    products derived from this software without specific prior written
+ *    permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+.L_memcpy_base:
+        // Assumes that n >= 0, and dst, src are valid pointers.
+        cmp     r2, #16
+        blo     .L_copy_less_than_16_unknown_align
+
+.L_copy_unknown_alignment:
+        // Unknown alignment of src and dst.
+        // Assumes that the first few bytes have already been prefetched.
+
+        // Align destination to 128 bits. The mainloop store instructions
+        // require this alignment or they will throw an exception.
+        rsb         r3, r0, #0
+        ands        r3, r3, #0xF
+        beq         2f
+
+        // Copy up to 15 bytes (count in r3).
+        sub         r2, r2, r3
+        movs        ip, r3, lsl #31
+
+        itt         mi
+        ldrbmi      lr, [r1], #1
+        strbmi      lr, [r0], #1
+        itttt       cs
+        ldrbcs      ip, [r1], #1
+        ldrbcs      lr, [r1], #1
+        strbcs      ip, [r0], #1
+        strbcs      lr, [r0], #1
+
+        movs        ip, r3, lsl #29
+        bge         1f
+        // Copies 4 bytes, dst 32 bits aligned before, at least 64 bits after.
+        vld4.8      {d0[0], d1[0], d2[0], d3[0]}, [r1]!
+        vst4.8      {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]!
+1:      bcc         2f
+        // Copies 8 bytes, dst 64 bits aligned before, at least 128 bits after.
+        vld1.8      {d0}, [r1]!
+        vst1.8      {d0}, [r0, :64]!
+
+2:      // Make sure we have at least 64 bytes to copy.
+        subs        r2, r2, #64
+        blo         2f
+
+1:      // The main loop copies 64 bytes at a time.
+        vld1.8      {d0  - d3},   [r1]!
+        vld1.8      {d4  - d7},   [r1]!
+        subs        r2, r2, #64
+        vstmia      r0!, {d0 - d7}
+        pld         [r1, #(64*10)]
+        bhs         1b
+
+2:      // Fix-up the remaining count and make sure we have >= 32 bytes left.
+        adds        r2, r2, #32
+        blo         3f
+
+        // 32 bytes. These cache lines were already preloaded.
+        vld1.8      {d0 - d3},  [r1]!
+        sub         r2, r2, #32
+        vst1.8      {d0 - d3},  [r0, :128]!
+3:      // Less than 32 left.
+        add         r2, r2, #32
+        tst         r2, #0x10
+        beq         .L_copy_less_than_16_unknown_align
+        // Copies 16 bytes, destination 128 bits aligned.
+        vld1.8      {d0, d1}, [r1]!
+        vst1.8      {d0, d1}, [r0, :128]!
+
+.L_copy_less_than_16_unknown_align:
+        // Copy up to 15 bytes (count in r2).
+        movs        ip, r2, lsl #29
+        bcc         1f
+        vld1.8      {d0}, [r1]!
+        vst1.8      {d0}, [r0]!
+1:      bge         2f
+        vld4.8      {d0[0], d1[0], d2[0], d3[0]}, [r1]!
+        vst4.8      {d0[0], d1[0], d2[0], d3[0]}, [r0]!
+
+2:      // Copy 0 to 4 bytes.
+        lsls        r2, r2, #31
+        itt         ne
+        ldrbne      lr, [r1], #1
+        strbne      lr, [r0], #1
+        itttt       cs
+        ldrbcs      ip, [r1], #1
+        ldrbcs      lr, [r1]
+        strbcs      ip, [r0], #1
+        strbcs      lr, [r0]
+
+        pop         {r0, pc}

libc/arch-arm/cortex-a53/cortex-a53.mk

@@ -1 +1,31 @@
-include bionic/libc/arch-arm/cortex-a7/cortex-a7.mk
+libc_openbsd_src_files_exclude_arm += \
+    upstream-openbsd/lib/libc/string/memmove.c \
+    upstream-openbsd/lib/libc/string/stpcpy.c \
+    upstream-openbsd/lib/libc/string/strcat.c \
+
+libc_bionic_src_files_exclude_arm += \
+    arch-arm/generic/bionic/memcpy.S \
+    arch-arm/generic/bionic/memset.S \
+    arch-arm/generic/bionic/strcmp.S \
+    arch-arm/generic/bionic/strcpy.S \
+    arch-arm/generic/bionic/strlen.c \
+    bionic/__strcat_chk.cpp \
+    bionic/__strcpy_chk.cpp \
+
+libc_bionic_src_files_arm += \
+    arch-arm/cortex-a53/bionic/memcpy.S \
+    arch-arm/cortex-a53/bionic/__strcat_chk.S \
+    arch-arm/cortex-a53/bionic/__strcpy_chk.S \
+
+libc_bionic_src_files_arm += \
+    arch-arm/cortex-a7/bionic/memset.S \
+
+libc_bionic_src_files_arm += \
+    arch-arm/cortex-a15/bionic/stpcpy.S \
+    arch-arm/cortex-a15/bionic/strcat.S \
+    arch-arm/cortex-a15/bionic/strcmp.S \
+    arch-arm/cortex-a15/bionic/strcpy.S \
+    arch-arm/cortex-a15/bionic/strlen.S \
+
+libc_bionic_src_files_arm += \
+    arch-arm/denver/bionic/memmove.S \

libc/arch-arm/cortex-a7/bionic/memset.S

@@ -0,0 +1,180 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/cpu-features.h>
+#include <private/bionic_asm.h>
+#include <private/libc_events.h>
+
+        /*
+         * Optimized memset() for ARM.
+         *
+         * memset() returns its first argument.
+         */
+
+        .fpu        neon
+        .syntax     unified
+
+ENTRY(__memset_chk)
+        cmp         r2, r3
+        bls         .L_done
+
+        // Preserve lr for backtrace.
+        push        {lr}
+        .cfi_def_cfa_offset 4
+        .cfi_rel_offset lr, 0
+
+        ldr         r0, error_message
+        ldr         r1, error_code
+1:
+        add         r0, pc
+        bl          __fortify_chk_fail
+error_code:
+        .word       BIONIC_EVENT_MEMSET_BUFFER_OVERFLOW
+error_message:
+        .word       error_string-(1b+8)
+END(__memset_chk)
+
+ENTRY(bzero)
+        mov         r2, r1
+        mov         r1, #0
+.L_done:
+        // Fall through to memset...
+END(bzero)
+
+ENTRY(memset)
+        mov         r3, r0
+        // At this point only d0, d1 are going to be used below.
+        vdup.8      q0, r1
+        cmp         r2, #16
+        blo         .L_set_less_than_16_unknown_align
+
+.L_check_alignment:
+        // Align destination to a double word to avoid the store crossing
+        // a cache line boundary.
+        ands        ip, r3, #7
+        bne         .L_do_double_word_align
+
+.L_double_word_aligned:
+        // Duplicate since the less than 64 can use d2, d3.
+        vmov        q1, q0
+        subs        r2, #64
+        blo         .L_set_less_than_64
+
+        // Duplicate the copy value so that we can store 64 bytes at a time.
+        vmov        q2, q0
+        vmov        q3, q0
+
+1:      // Main loop stores 64 bytes at a time.
+        subs        r2, #64
+        vstmia      r3!, {d0 - d7}
+        bge         1b
+
+.L_set_less_than_64:
+        // Restore r2 to the count of bytes left to set.
+        add         r2, #64
+        lsls        ip, r2, #27
+        bcc         .L_set_less_than_32
+        // Set 32 bytes.
+        vstmia      r3!, {d0 - d3}
+
+.L_set_less_than_32:
+        bpl         .L_set_less_than_16
+        // Set 16 bytes.
+        vstmia      r3!, {d0, d1}
+
+.L_set_less_than_16:
+        // Less than 16 bytes to set.
+        lsls        ip, r2, #29
+        bcc         .L_set_less_than_8
+
+        // Set 8 bytes.
+        vstmia      r3!, {d0}
+
+.L_set_less_than_8:
+        bpl         .L_set_less_than_4
+        // Set 4 bytes
+        vst1.32     {d0[0]}, [r3]!
+
+.L_set_less_than_4:
+        lsls        ip, r2, #31
+        it          ne
+        strbne      r1, [r3], #1
+        itt         cs
+        strbcs      r1, [r3], #1
+        strbcs      r1, [r3]
+        bx          lr
+
+.L_do_double_word_align:
+        rsb         ip, ip, #8
+        sub         r2, r2, ip
+
+        // Do this comparison now, otherwise we'll need to save a
+        // register to the stack since we've used all available
+        // registers.
+        cmp         ip, #4
+        blo         1f
+
+        // Need to do a four byte copy.
+        movs        ip, ip, lsl #31
+        it          mi
+        strbmi      r1, [r3], #1
+        itt         cs
+        strbcs      r1, [r3], #1
+        strbcs      r1, [r3], #1
+        vst1.32     {d0[0]}, [r3]!
+        b           .L_double_word_aligned
+
+1:
+        // No four byte copy.
+        movs        ip, ip, lsl #31
+        it          mi
+        strbmi      r1, [r3], #1
+        itt         cs
+        strbcs      r1, [r3], #1
+        strbcs      r1, [r3], #1
+        b           .L_double_word_aligned
+
+.L_set_less_than_16_unknown_align:
+        // Set up to 15 bytes.
+        movs        ip, r2, lsl #29
+        bcc         1f
+        vst1.8      {d0}, [r3]!
+1:      bge         2f
+        vst1.32     {d0[0]}, [r3]!
+2:      movs        ip, r2, lsl #31
+        it          mi
+        strbmi      r1, [r3], #1
+        itt         cs
+        strbcs      r1, [r3], #1
+        strbcs      r1, [r3], #1
+        bx          lr
+END(memset)
+
+        .data
+error_string:
+        .string     "memset: prevented write past end of buffer"

libc/arch-arm/cortex-a7/cortex-a7.mk

@@ -1 +1,29 @@
-include bionic/libc/arch-arm/cortex-a15/cortex-a15.mk
+libc_openbsd_src_files_exclude_arm += \
+    upstream-openbsd/lib/libc/string/memmove.c \
+    upstream-openbsd/lib/libc/string/stpcpy.c \
+    upstream-openbsd/lib/libc/string/strcat.c \
+
+libc_bionic_src_files_exclude_arm += \
+    arch-arm/generic/bionic/memcpy.S \
+    arch-arm/generic/bionic/memset.S \
+    arch-arm/generic/bionic/strcmp.S \
+    arch-arm/generic/bionic/strcpy.S \
+    arch-arm/generic/bionic/strlen.c \
+    bionic/__strcat_chk.cpp \
+    bionic/__strcpy_chk.cpp \
+
+libc_bionic_src_files_arm += \
+    arch-arm/cortex-a7/bionic/memset.S \
+
+libc_bionic_src_files_arm += \
+    arch-arm/cortex-a15/bionic/memcpy.S \
+    arch-arm/cortex-a15/bionic/stpcpy.S \
+    arch-arm/cortex-a15/bionic/strcat.S \
+    arch-arm/cortex-a15/bionic/__strcat_chk.S \
+    arch-arm/cortex-a15/bionic/strcmp.S \
+    arch-arm/cortex-a15/bionic/strcpy.S \
+    arch-arm/cortex-a15/bionic/__strcpy_chk.S \
+    arch-arm/cortex-a15/bionic/strlen.S \
+
+libc_bionic_src_files_arm += \
+    arch-arm/denver/bionic/memmove.S \

libc/arch-arm/cortex-a9/bionic/memcpy_base.S

@@ -133,8 +133,7 @@ ENTRY_PRIVATE(MEMCPY_BASE)
         strbcs      ip, [r0], #1
         strbcs      lr, [r0], #1
 
-        ldmfd       sp!, {r0, lr}
-        bx          lr
+        ldmfd       sp!, {r0, pc}
 END(MEMCPY_BASE)
 
 ENTRY_PRIVATE(MEMCPY_BASE_ALIGNED)

libc/arch-arm/cortex-a9/bionic/memset.S

@@ -69,12 +69,9 @@ END(bzero)
 ENTRY(memset)
         // The neon memset only wins for less than 132.
         cmp         r2, #132
-        bhi         __memset_large_copy
-
-        stmfd       sp!, {r0}
-        .cfi_def_cfa_offset 4
-        .cfi_rel_offset r0, 0
+        bhi         .L_memset_large_copy
 
+        mov         r3, r0
         vdup.8      q0, r1
 
         /* make sure we have at least 32 bytes to write */
@@ -84,7 +81,7 @@ ENTRY(memset)
 
 1:      /* The main loop writes 32 bytes at a time */
         subs        r2, r2, #32
-        vst1.8      {d0 - d3}, [r0]!
+        vst1.8      {d0 - d3}, [r3]!
         bhs         1b
 
 2:      /* less than 32 left */
@@ -93,22 +90,20 @@ ENTRY(memset)
         beq         3f
 
         // writes 16 bytes, 128-bits aligned
-        vst1.8      {d0, d1}, [r0]!
+        vst1.8      {d0, d1}, [r3]!
 3:      /* write up to 15-bytes (count in r2) */
         movs        ip, r2, lsl #29
         bcc         1f
-        vst1.8      {d0}, [r0]!
+        vst1.8      {d0}, [r3]!
 1:      bge         2f
-        vst1.32     {d0[0]}, [r0]!
+        vst1.32     {d0[0]}, [r3]!
 2:      movs        ip, r2, lsl #31
-        strbmi      r1, [r0], #1
-        strbcs      r1, [r0], #1
-        strbcs      r1, [r0], #1
-        ldmfd       sp!, {r0}
+        strbmi      r1, [r3], #1
+        strbcs      r1, [r3], #1
+        strbcs      r1, [r3], #1
         bx          lr
-END(memset)
 
-ENTRY_PRIVATE(__memset_large_copy)
+.L_memset_large_copy:
         /* compute the offset to align the destination
          * offset = (4-(src&3))&3 = -src & 3
          */
@@ -136,8 +131,7 @@ ENTRY_PRIVATE(__memset_large_copy)
         strbcs      r1, [r0], #1
         strbmi      r1, [r0], #1
         subs        r2, r2, r3
-        popls       {r0, r4-r7, lr}   /* return */
-        bxls        lr
+        popls       {r0, r4-r7, pc}   /* return */
 
         /* align the destination to a cache-line */
         mov         r12, r1
@@ -180,9 +174,8 @@ ENTRY_PRIVATE(__memset_large_copy)
         strhmi      r1, [r0], #2
         movs        r2, r2, lsl #2
         strbcs      r1, [r0]
-        ldmfd       sp!, {r0, r4-r7, lr}
-        bx          lr
-END(__memset_large_copy)
+        ldmfd       sp!, {r0, r4-r7, pc}
+END(memset)
 
         .data
 error_string:

libc/arch-arm/cortex-a9/bionic/strcat.S

@@ -70,7 +70,7 @@
 
     .macro m_scan_byte
     ldrb    r3, [r0]
-    cbz     r3, strcat_r0_scan_done
+    cbz     r3, .Lstrcat_r0_scan_done
     add     r0, #1
     .endm // m_scan_byte
 
@@ -84,10 +84,10 @@ ENTRY(strcat)
     // Quick check to see if src is empty.
     ldrb        r2, [r1]
     pld         [r1, #0]
-    cbnz        r2, strcat_continue
+    cbnz        r2, .Lstrcat_continue
     bx          lr
 
-strcat_continue:
+.Lstrcat_continue:
     // To speed up really small dst strings, unroll checking the first 4 bytes.
     m_push
     m_scan_byte
@@ -96,10 +96,10 @@ strcat_continue:
     m_scan_byte
 
     ands    r3, r0, #7
-    bne     strcat_align_src
+    bne     .Lstrcat_align_src
 
     .p2align 2
-strcat_mainloop:
+.Lstrcat_mainloop:
     ldmia   r0!, {r2, r3}
 
     pld     [r0, #64]
@@ -107,28 +107,28 @@ strcat_mainloop:
     sub     ip, r2, #0x01010101
     bic     ip, ip, r2
     ands    ip, ip, #0x80808080
-    bne     strcat_zero_in_first_register
+    bne     .Lstrcat_zero_in_first_register
 
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     strcat_zero_in_second_register
-    b       strcat_mainloop
+    bne     .Lstrcat_zero_in_second_register
+    b       .Lstrcat_mainloop
 
-strcat_zero_in_first_register:
+.Lstrcat_zero_in_first_register:
     sub     r0, r0, #4
 
-strcat_zero_in_second_register:
+.Lstrcat_zero_in_second_register:
     // Check for zero in byte 0.
     tst     ip, #0x80
     it      ne
     subne   r0, r0, #4
-    bne     strcat_r0_scan_done
+    bne     .Lstrcat_r0_scan_done
     // Check for zero in byte 1.
     tst     ip, #0x8000
     it      ne
     subne   r0, r0, #3
-    bne     strcat_r0_scan_done
+    bne     .Lstrcat_r0_scan_done
     // Check for zero in byte 2.
     tst     ip, #0x800000
     it      ne
@@ -137,33 +137,33 @@ strcat_zero_in_second_register:
     // Zero is in byte 3.
     subeq   r0, r0, #1
 
-strcat_r0_scan_done:
+.Lstrcat_r0_scan_done:
     // Unroll the first 8 bytes that will be copied.
-    m_copy_byte reg=r2, cmd=cbz, label=strcpy_finish
-    m_copy_byte reg=r3, cmd=cbz, label=strcpy_finish
-    m_copy_byte reg=r4, cmd=cbz, label=strcpy_finish
-    m_copy_byte reg=r5, cmd=cbz, label=strcpy_finish
-    m_copy_byte reg=r2, cmd=cbz, label=strcpy_finish
-    m_copy_byte reg=r3, cmd=cbz, label=strcpy_finish
-    m_copy_byte reg=r4, cmd=cbz, label=strcpy_finish
-    m_copy_byte reg=r5, cmd=cbnz, label=strcpy_continue
+    m_copy_byte reg=r2, cmd=cbz, label=.Lstrcpy_finish
+    m_copy_byte reg=r3, cmd=cbz, label=.Lstrcpy_finish
+    m_copy_byte reg=r4, cmd=cbz, label=.Lstrcpy_finish
+    m_copy_byte reg=r5, cmd=cbz, label=.Lstrcpy_finish
+    m_copy_byte reg=r2, cmd=cbz, label=.Lstrcpy_finish
+    m_copy_byte reg=r3, cmd=cbz, label=.Lstrcpy_finish
+    m_copy_byte reg=r4, cmd=cbz, label=.Lstrcpy_finish
+    m_copy_byte reg=r5, cmd=cbnz, label=.Lstrcpy_continue
 
-strcpy_finish:
+.Lstrcpy_finish:
     m_ret   inst=pop
 
-strcpy_continue:
+.Lstrcpy_continue:
     pld     [r1, #0]
     ands    r3, r0, #7
-    bne     strcpy_align_dst
+    bne     .Lstrcpy_align_dst
 
-strcpy_check_src_align:
+.Lstrcpy_check_src_align:
     // At this point dst is aligned to a double word, check if src
     // is also aligned to a double word.
     ands    r3, r1, #7
-    bne     strcpy_unaligned_copy
+    bne     .Lstrcpy_unaligned_copy
 
     .p2align 2
-strcpy_mainloop:
+.Lstrcpy_mainloop:
     ldmia   r1!, {r2, r3}
 
     pld     [r1, #64]
@@ -171,17 +171,17 @@ strcpy_mainloop:
     sub     ip, r2, #0x01010101
     bic     ip, ip, r2
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_first_register
+    bne     .Lstrcpy_zero_in_first_register
 
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_second_register
+    bne     .Lstrcpy_zero_in_second_register
 
     stmia   r0!, {r2, r3}
-    b       strcpy_mainloop
+    b       .Lstrcpy_mainloop
 
-strcpy_zero_in_first_register:
+.Lstrcpy_zero_in_first_register:
     lsls    lr, ip, #17
     itt     ne
     strbne  r2, [r0]
@@ -198,7 +198,7 @@ strcpy_zero_in_first_register:
     strb    r3, [r0]
     m_ret   inst=pop
 
-strcpy_zero_in_second_register:
+.Lstrcpy_zero_in_second_register:
     lsls    lr, ip, #17
     ittt    ne
     stmiane r0!, {r2}
@@ -218,18 +218,18 @@ strcpy_zero_in_second_register:
     strb    r4, [r0]
     m_ret   inst=pop
 
-strcpy_align_dst:
+.Lstrcpy_align_dst:
     // Align to a double word (64 bits).
     rsb     r3, r3, #8
     lsls    ip, r3, #31
-    beq     strcpy_align_to_32
+    beq     .Lstrcpy_align_to_32
 
     ldrb    r2, [r1], #1
     strb    r2, [r0], #1
-    cbz     r2, strcpy_complete
+    cbz     r2, .Lstrcpy_complete
 
-strcpy_align_to_32:
-    bcc     strcpy_align_to_64
+.Lstrcpy_align_to_32:
+    bcc     .Lstrcpy_align_to_64
 
     ldrb    r4, [r1], #1
     strb    r4, [r0], #1
@@ -242,76 +242,83 @@ strcpy_align_to_32:
     it      eq
     m_ret   inst=popeq
 
-strcpy_align_to_64:
+.Lstrcpy_align_to_64:
     tst     r3, #4
-    beq     strcpy_check_src_align
-    ldr     r2, [r1], #4
+    beq     .Lstrcpy_check_src_align
+    // Read one byte at a time since we don't know the src alignment
+    // and we don't want to read into a different page.
+    ldrb    r4, [r1], #1
+    strb    r4, [r0], #1
+    cbz     r4, .Lstrcpy_complete
+    ldrb    r5, [r1], #1
+    strb    r5, [r0], #1
+    cbz     r5, .Lstrcpy_complete
+    ldrb    r4, [r1], #1
+    strb    r4, [r0], #1
+    cbz     r4, .Lstrcpy_complete
+    ldrb    r5, [r1], #1
+    strb    r5, [r0], #1
+    cbz     r5, .Lstrcpy_complete
+    b       .Lstrcpy_check_src_align
 
-    sub     ip, r2, #0x01010101
-    bic     ip, ip, r2
-    ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_first_register
-    stmia   r0!, {r2}
-    b       strcpy_check_src_align
-
-strcpy_complete:
+.Lstrcpy_complete:
     m_ret   inst=pop
 
-strcpy_unaligned_copy:
+.Lstrcpy_unaligned_copy:
     // Dst is aligned to a double word, while src is at an unknown alignment.
     // There are 7 different versions of the unaligned copy code
     // to prevent overreading the src. The mainloop of every single version
     // will store 64 bits per loop. The difference is how much of src can
     // be read without potentially crossing a page boundary.
     tbb     [pc, r3]
-strcpy_unaligned_branchtable:
+.Lstrcpy_unaligned_branchtable:
     .byte 0
-    .byte ((strcpy_unalign7 - strcpy_unaligned_branchtable)/2)
-    .byte ((strcpy_unalign6 - strcpy_unaligned_branchtable)/2)
-    .byte ((strcpy_unalign5 - strcpy_unaligned_branchtable)/2)
-    .byte ((strcpy_unalign4 - strcpy_unaligned_branchtable)/2)
-    .byte ((strcpy_unalign3 - strcpy_unaligned_branchtable)/2)
-    .byte ((strcpy_unalign2 - strcpy_unaligned_branchtable)/2)
-    .byte ((strcpy_unalign1 - strcpy_unaligned_branchtable)/2)
+    .byte ((.Lstrcpy_unalign7 - .Lstrcpy_unaligned_branchtable)/2)
+    .byte ((.Lstrcpy_unalign6 - .Lstrcpy_unaligned_branchtable)/2)
+    .byte ((.Lstrcpy_unalign5 - .Lstrcpy_unaligned_branchtable)/2)
+    .byte ((.Lstrcpy_unalign4 - .Lstrcpy_unaligned_branchtable)/2)
+    .byte ((.Lstrcpy_unalign3 - .Lstrcpy_unaligned_branchtable)/2)
+    .byte ((.Lstrcpy_unalign2 - .Lstrcpy_unaligned_branchtable)/2)
+    .byte ((.Lstrcpy_unalign1 - .Lstrcpy_unaligned_branchtable)/2)
 
     .p2align 2
     // Can read 7 bytes before possibly crossing a page.
-strcpy_unalign7:
+.Lstrcpy_unalign7:
     ldr     r2, [r1], #4
 
     sub     ip, r2, #0x01010101
     bic     ip, ip, r2
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_first_register
+    bne     .Lstrcpy_zero_in_first_register
 
     ldrb    r3, [r1]
-    cbz     r3, strcpy_unalign7_copy5bytes
+    cbz     r3, .Lstrcpy_unalign7_copy5bytes
     ldrb    r4, [r1, #1]
-    cbz     r4, strcpy_unalign7_copy6bytes
+    cbz     r4, .Lstrcpy_unalign7_copy6bytes
     ldrb    r5, [r1, #2]
-    cbz     r5, strcpy_unalign7_copy7bytes
+    cbz     r5, .Lstrcpy_unalign7_copy7bytes
 
     ldr     r3, [r1], #4
     pld     [r1, #64]
 
     lsrs    ip, r3, #24
     stmia   r0!, {r2, r3}
-    beq     strcpy_unalign_return
-    b       strcpy_unalign7
+    beq     .Lstrcpy_unalign_return
+    b       .Lstrcpy_unalign7
 
-strcpy_unalign7_copy5bytes:
+.Lstrcpy_unalign7_copy5bytes:
     stmia   r0!, {r2}
     strb    r3, [r0]
-strcpy_unalign_return:
+.Lstrcpy_unalign_return:
     m_ret   inst=pop
 
-strcpy_unalign7_copy6bytes:
+.Lstrcpy_unalign7_copy6bytes:
     stmia   r0!, {r2}
     strb    r3, [r0], #1
     strb    r4, [r0], #1
     m_ret   inst=pop
 
-strcpy_unalign7_copy7bytes:
+.Lstrcpy_unalign7_copy7bytes:
     stmia   r0!, {r2}
     strb    r3, [r0], #1
     strb    r4, [r0], #1
@@ -320,30 +327,30 @@ strcpy_unalign7_copy7bytes:
 
     .p2align 2
     // Can read 6 bytes before possibly crossing a page.
-strcpy_unalign6:
+.Lstrcpy_unalign6:
     ldr     r2, [r1], #4
 
     sub     ip, r2, #0x01010101
     bic     ip, ip, r2
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_first_register
+    bne     .Lstrcpy_zero_in_first_register
 
     ldrb    r4, [r1]
-    cbz     r4, strcpy_unalign_copy5bytes
+    cbz     r4, .Lstrcpy_unalign_copy5bytes
     ldrb    r5, [r1, #1]
-    cbz     r5, strcpy_unalign_copy6bytes
+    cbz     r5, .Lstrcpy_unalign_copy6bytes
 
     ldr     r3, [r1], #4
     pld     [r1, #64]
 
     tst     r3, #0xff0000
-    beq     strcpy_unalign6_copy7bytes
+    beq     .Lstrcpy_unalign6_copy7bytes
     lsrs    ip, r3, #24
     stmia   r0!, {r2, r3}
-    beq     strcpy_unalign_return
-    b       strcpy_unalign6
+    beq     .Lstrcpy_unalign_return
+    b       .Lstrcpy_unalign6
 
-strcpy_unalign6_copy7bytes:
+.Lstrcpy_unalign6_copy7bytes:
     stmia   r0!, {r2}
     strh    r3, [r0], #2
     lsr     r3, #16
@@ -352,16 +359,16 @@ strcpy_unalign6_copy7bytes:
 
     .p2align 2
     // Can read 5 bytes before possibly crossing a page.
-strcpy_unalign5:
+.Lstrcpy_unalign5:
     ldr     r2, [r1], #4
 
     sub     ip, r2, #0x01010101
     bic     ip, ip, r2
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_first_register
+    bne     .Lstrcpy_zero_in_first_register
 
     ldrb    r4, [r1]
-    cbz     r4, strcpy_unalign_copy5bytes
+    cbz     r4, .Lstrcpy_unalign_copy5bytes
 
     ldr     r3, [r1], #4
 
@@ -370,17 +377,17 @@ strcpy_unalign5:
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_second_register
+    bne     .Lstrcpy_zero_in_second_register
 
     stmia   r0!, {r2, r3}
-    b       strcpy_unalign5
+    b       .Lstrcpy_unalign5
 
-strcpy_unalign_copy5bytes:
+.Lstrcpy_unalign_copy5bytes:
     stmia   r0!, {r2}
     strb    r4, [r0]
     m_ret   inst=pop
 
-strcpy_unalign_copy6bytes:
+.Lstrcpy_unalign_copy6bytes:
     stmia   r0!, {r2}
     strb    r4, [r0], #1
     strb    r5, [r0]
@@ -388,13 +395,13 @@ strcpy_unalign_copy6bytes:
 
     .p2align 2
     // Can read 4 bytes before possibly crossing a page.
-strcpy_unalign4:
+.Lstrcpy_unalign4:
     ldmia   r1!, {r2}
 
     sub     ip, r2, #0x01010101
     bic     ip, ip, r2
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_first_register
+    bne     .Lstrcpy_zero_in_first_register
 
     ldmia   r1!, {r3}
     pld     [r1, #64]
@@ -402,20 +409,20 @@ strcpy_unalign4:
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_second_register
+    bne     .Lstrcpy_zero_in_second_register
 
     stmia   r0!, {r2, r3}
-    b       strcpy_unalign4
+    b       .Lstrcpy_unalign4
 
     .p2align 2
     // Can read 3 bytes before possibly crossing a page.
-strcpy_unalign3:
+.Lstrcpy_unalign3:
     ldrb    r2, [r1]
-    cbz     r2, strcpy_unalign3_copy1byte
+    cbz     r2, .Lstrcpy_unalign3_copy1byte
     ldrb    r3, [r1, #1]
-    cbz     r3, strcpy_unalign3_copy2bytes
+    cbz     r3, .Lstrcpy_unalign3_copy2bytes
     ldrb    r4, [r1, #2]
-    cbz     r4, strcpy_unalign3_copy3bytes
+    cbz     r4, .Lstrcpy_unalign3_copy3bytes
 
     ldr     r2, [r1], #4
     ldr     r3, [r1], #4
@@ -423,26 +430,26 @@ strcpy_unalign3:
     pld     [r1, #64]
 
     lsrs    lr, r2, #24
-    beq     strcpy_unalign_copy4bytes
+    beq     .Lstrcpy_unalign_copy4bytes
 
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_second_register
+    bne     .Lstrcpy_zero_in_second_register
 
     stmia   r0!, {r2, r3}
-    b       strcpy_unalign3
+    b       .Lstrcpy_unalign3
 
-strcpy_unalign3_copy1byte:
+.Lstrcpy_unalign3_copy1byte:
     strb    r2, [r0]
     m_ret   inst=pop
 
-strcpy_unalign3_copy2bytes:
+.Lstrcpy_unalign3_copy2bytes:
     strb    r2, [r0], #1
     strb    r3, [r0]
     m_ret   inst=pop
 
-strcpy_unalign3_copy3bytes:
+.Lstrcpy_unalign3_copy3bytes:
     strb    r2, [r0], #1
     strb    r3, [r0], #1
     strb    r4, [r0]
@@ -450,34 +457,34 @@ strcpy_unalign3_copy3bytes:
 
     .p2align 2
     // Can read 2 bytes before possibly crossing a page.
-strcpy_unalign2:
+.Lstrcpy_unalign2:
     ldrb    r2, [r1]
-    cbz     r2, strcpy_unalign_copy1byte
+    cbz     r2, .Lstrcpy_unalign_copy1byte
     ldrb    r3, [r1, #1]
-    cbz     r3, strcpy_unalign_copy2bytes
+    cbz     r3, .Lstrcpy_unalign_copy2bytes
 
     ldr     r2, [r1], #4
     ldr     r3, [r1], #4
     pld     [r1, #64]
 
     tst     r2, #0xff0000
-    beq     strcpy_unalign_copy3bytes
+    beq     .Lstrcpy_unalign_copy3bytes
     lsrs    ip, r2, #24
-    beq     strcpy_unalign_copy4bytes
+    beq     .Lstrcpy_unalign_copy4bytes
 
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_second_register
+    bne     .Lstrcpy_zero_in_second_register
 
     stmia   r0!, {r2, r3}
-    b       strcpy_unalign2
+    b       .Lstrcpy_unalign2
 
     .p2align 2
     // Can read 1 byte before possibly crossing a page.
-strcpy_unalign1:
+.Lstrcpy_unalign1:
     ldrb    r2, [r1]
-    cbz     r2, strcpy_unalign_copy1byte
+    cbz     r2, .Lstrcpy_unalign_copy1byte
 
     ldr     r2, [r1], #4
     ldr     r3, [r1], #4
@@ -487,62 +494,62 @@ strcpy_unalign1:
     sub     ip, r2, #0x01010101
     bic     ip, ip, r2
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_first_register
+    bne     .Lstrcpy_zero_in_first_register
 
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     strcpy_zero_in_second_register
+    bne     .Lstrcpy_zero_in_second_register
 
     stmia   r0!, {r2, r3}
-    b       strcpy_unalign1
+    b       .Lstrcpy_unalign1
 
-strcpy_unalign_copy1byte:
+.Lstrcpy_unalign_copy1byte:
     strb    r2, [r0]
     m_ret   inst=pop
 
-strcpy_unalign_copy2bytes:
+.Lstrcpy_unalign_copy2bytes:
     strb    r2, [r0], #1
     strb    r3, [r0]
     m_ret   inst=pop
 
-strcpy_unalign_copy3bytes:
+.Lstrcpy_unalign_copy3bytes:
     strh    r2, [r0], #2
     lsr     r2, #16
     strb    r2, [r0]
     m_ret   inst=pop
 
-strcpy_unalign_copy4bytes:
+.Lstrcpy_unalign_copy4bytes:
     stmia   r0, {r2}
     m_ret   inst=pop
 
-strcat_align_src:
+.Lstrcat_align_src:
     // Align to a double word (64 bits).
     rsb     r3, r3, #8
     lsls    ip, r3, #31
-    beq     strcat_align_to_32
+    beq     .Lstrcat_align_to_32
     ldrb    r2, [r0], #1
-    cbz     r2, strcat_r0_update
+    cbz     r2, .Lstrcat_r0_update
 
-strcat_align_to_32:
-    bcc     strcat_align_to_64
+.Lstrcat_align_to_32:
+    bcc     .Lstrcat_align_to_64
     ldrb    r2, [r0], #1
-    cbz     r2, strcat_r0_update
+    cbz     r2, .Lstrcat_r0_update
     ldrb    r2, [r0], #1
-    cbz     r2, strcat_r0_update
+    cbz     r2, .Lstrcat_r0_update
 
-strcat_align_to_64:
+.Lstrcat_align_to_64:
     tst     r3, #4
-    beq     strcat_mainloop
+    beq     .Lstrcat_mainloop
     ldr     r3, [r0], #4
 
     sub     ip, r3, #0x01010101
     bic     ip, ip, r3
     ands    ip, ip, #0x80808080
-    bne     strcat_zero_in_second_register
-    b       strcat_mainloop
+    bne     .Lstrcat_zero_in_second_register
+    b       .Lstrcat_mainloop
 
-strcat_r0_update:
+.Lstrcat_r0_update:
     sub     r0, r0, #1
-    b strcat_r0_scan_done
+    b .Lstrcat_r0_scan_done
 END(strcat)

libc/arch-arm/cortex-a9/bionic/string_copy.S

@@ -244,13 +244,20 @@ ENTRY(strcpy)
 .Lstringcopy_align_to_64:
     tst     r3, #4
     beq     .Lstringcopy_check_src_align
-    ldr     r2, [r1], #4
-
-    sub     ip, r2, #0x01010101
-    bic     ip, ip, r2
-    ands    ip, ip, #0x80808080
-    bne     .Lstringcopy_zero_in_first_register
-    stmia   r0!, {r2}
+    // Read one byte at a time since we don't have any idea about the alignment
+    // of the source and we don't want to read into a different page.
+    ldrb    r2, [r1], #1
+    strb    r2, [r0], #1
+    cbz     r2, .Lstringcopy_complete
+    ldrb    r2, [r1], #1
+    strb    r2, [r0], #1
+    cbz     r2, .Lstringcopy_complete
+    ldrb    r2, [r1], #1
+    strb    r2, [r0], #1
+    cbz     r2, .Lstringcopy_complete
+    ldrb    r2, [r1], #1
+    strb    r2, [r0], #1
+    cbz     r2, .Lstringcopy_complete
     b       .Lstringcopy_check_src_align
 
 .Lstringcopy_complete:

libc/arch-arm/cortex-a9/cortex-a9.mk

@@ -1,3 +1,18 @@
+libc_openbsd_src_files_exclude_arm += \
+    upstream-openbsd/lib/libc/string/memmove.c \
+    upstream-openbsd/lib/libc/string/stpcpy.c \
+    upstream-openbsd/lib/libc/string/strcat.c \
+    upstream-openbsd/lib/libc/string/strcpy.c \
+
+libc_bionic_src_files_exclude_arm += \
+    arch-arm/generic/bionic/memcpy.S \
+    arch-arm/generic/bionic/memset.S \
+    arch-arm/generic/bionic/strcmp.S \
+    arch-arm/generic/bionic/strcpy.S \
+    arch-arm/generic/bionic/strlen.c \
+    bionic/__strcat_chk.cpp \
+    bionic/__strcpy_chk.cpp \
+
 libc_bionic_src_files_arm += \
     arch-arm/cortex-a9/bionic/memcpy.S \
     arch-arm/cortex-a9/bionic/memset.S \
@@ -9,8 +24,5 @@ libc_bionic_src_files_arm += \
     arch-arm/cortex-a9/bionic/__strcpy_chk.S \
     arch-arm/cortex-a9/bionic/strlen.S \
 
-libc_bionic_src_files_arm += \
-    arch-arm/generic/bionic/memcmp.S \
-
 libc_bionic_src_files_arm += \
     arch-arm/denver/bionic/memmove.S \

libc/arch-arm/denver/denver.mk

@@ -1,5 +1,18 @@
+libc_openbsd_src_files_exclude_arm += \
+    upstream-openbsd/lib/libc/string/memmove.c \
+    upstream-openbsd/lib/libc/string/stpcpy.c \
+    upstream-openbsd/lib/libc/string/strcat.c \
+
+libc_bionic_src_files_exclude_arm += \
+    arch-arm/generic/bionic/memcpy.S \
+    arch-arm/generic/bionic/memset.S \
+    arch-arm/generic/bionic/strcmp.S \
+    arch-arm/generic/bionic/strcpy.S \
+    arch-arm/generic/bionic/strlen.c \
+    bionic/__strcat_chk.cpp \
+    bionic/__strcpy_chk.cpp \
+
 libc_bionic_src_files_arm += \
-    arch-arm/generic/bionic/memcmp.S \
     arch-arm/denver/bionic/memcpy.S \
     arch-arm/denver/bionic/memmove.S \
     arch-arm/denver/bionic/memset.S \

libc/arch-arm/generic/bionic/memcmp.S

@@ -221,8 +221,7 @@ ENTRY(memcmp)
         bne         8b
 
 9:      /* restore registers and return */
-        ldmfd       sp!, {r4, lr}
-        bx          lr
+        ldmfd       sp!, {r4, pc}
 
 10:     /* process less than 12 bytes */
         cmp         r2, #0

libc/arch-arm/generic/bionic/memcpy.S

@@ -194,8 +194,7 @@ ENTRY(memcpy)
 
         /* we're done! restore everything and return */
 1:      ldmfd       sp!, {r5-r11}
-        ldmfd       sp!, {r0, r4, lr}
-        bx          lr
+        ldmfd       sp!, {r0, r4, pc}
 
         /********************************************************************/
 
@@ -385,8 +384,7 @@ ENTRY(memcpy)
 
         /* we're done! restore sp and spilled registers and return */
         add         sp,  sp, #28
-        ldmfd       sp!, {r0, r4, lr}
-        bx          lr
+        ldmfd       sp!, {r0, r4, pc}
 END(memcpy)
 
         // Only reached when the __memcpy_chk check fails.

libc/arch-arm/generic/bionic/memset.S

@@ -82,8 +82,7 @@ ENTRY(memset)
         strbcs      r1, [r0], #1
         strbmi      r1, [r0], #1
         subs        r2, r2, r3
-        popls       {r0, r4-r7, lr}    /* return */
-        bxls        lr
+        popls       {r0, r4-r7, pc}    /* return */
 
         /* align the destination to a cache-line */
         mov         r12, r1
@@ -126,8 +125,7 @@ ENTRY(memset)
         strhmi      r1, [r0], #2
         movs        r2, r2, lsl #2
         strbcs      r1, [r0]
-        ldmfd       sp!, {r0, r4-r7, lr}
-        bx          lr
+        ldmfd       sp!, {r0, r4-r7, pc}
 END(memset)
 
         .data

libc/arch-arm/generic/generic.mk

@@ -1,14 +0,0 @@
-libc_bionic_src_files_arm += \
-    arch-arm/generic/bionic/memcmp.S \
-    arch-arm/generic/bionic/memcpy.S \
-    arch-arm/generic/bionic/memset.S \
-    arch-arm/generic/bionic/strcmp.S \
-    arch-arm/generic/bionic/strcpy.S \
-    arch-arm/generic/bionic/strlen.c \
-    bionic/__strcat_chk.cpp \
-    bionic/__strcpy_chk.cpp \
-
-libc_openbsd_src_files_arm += \
-    upstream-openbsd/lib/libc/string/memmove.c \
-    upstream-openbsd/lib/libc/string/stpcpy.c \
-    upstream-openbsd/lib/libc/string/strcat.c \

libc/arch-arm/krait/bionic/__strcat_chk.S

@@ -40,7 +40,7 @@
 ENTRY(__strcat_chk)
     pld     [r0, #0]
     push    {r0, lr}
-    .cfi_def_cfa_offset 8
+    .cfi_adjust_cfa_offset 8
     .cfi_rel_offset r0, 0
     .cfi_rel_offset lr, 4
     push    {r4, r5}
@@ -177,7 +177,7 @@ ENTRY(__strcat_chk)
 .L_strlen_done:
     add     r2, r3, r4
     cmp     r2, lr
-    bhi     __strcat_chk_failed
+    bhi     .L_strcat_chk_failed
 
     // Set up the registers for the memcpy code.
     mov     r1, r5
@@ -185,20 +185,17 @@ ENTRY(__strcat_chk)
     mov     r2, r4
     add     r0, r0, r3
     pop     {r4, r5}
-END(__strcat_chk)
+    .cfi_adjust_cfa_offset -8
+    .cfi_restore r4
+    .cfi_restore r5
 
-#define MEMCPY_BASE         __strcat_chk_memcpy_base
-#define MEMCPY_BASE_ALIGNED __strcat_chk_memcpy_base_aligned
 #include "memcpy_base.S"
 
-ENTRY_PRIVATE(__strcat_chk_failed)
-    .cfi_def_cfa_offset 8
-    .cfi_rel_offset r0, 0
-    .cfi_rel_offset lr, 4
+    // Undo the above cfi directives.
     .cfi_adjust_cfa_offset 8
     .cfi_rel_offset r4, 0
     .cfi_rel_offset r5, 4
-
+.L_strcat_chk_failed:
     ldr     r0, error_message
     ldr     r1, error_code
 1:
@@ -208,7 +205,7 @@ error_code:
     .word   BIONIC_EVENT_STRCAT_BUFFER_OVERFLOW
 error_message:
     .word   error_string-(1b+4)
-END(__strcat_chk_failed)
+END(__strcat_chk)
 
     .data
 error_string:

libc/arch-arm/krait/bionic/__strcpy_chk.S

@@ -39,7 +39,7 @@
 ENTRY(__strcpy_chk)
     pld     [r0, #0]
     push    {r0, lr}
-    .cfi_def_cfa_offset 8
+    .cfi_adjust_cfa_offset 8
     .cfi_rel_offset r0, 0
     .cfi_rel_offset lr, 4
 
@@ -149,21 +149,14 @@ ENTRY(__strcpy_chk)
     pld     [r1, #64]
     ldr     r0, [sp]
     cmp     r3, lr
-    bhs     __strcpy_chk_failed
+    bhs     .L_strcpy_chk_failed
 
     // Add 1 for copy length to get the string terminator.
     add     r2, r3, #1
-END(__strcpy_chk)
 
-#define MEMCPY_BASE         __strcpy_chk_memcpy_base
-#define MEMCPY_BASE_ALIGNED __strcpy_chk_memcpy_base_aligned
 #include "memcpy_base.S"
 
-ENTRY_PRIVATE(__strcpy_chk_failed)
-    .cfi_def_cfa_offset 8
-    .cfi_rel_offset r0, 0
-    .cfi_rel_offset lr, 4
-
+.L_strcpy_chk_failed:
     ldr     r0, error_message
     ldr     r1, error_code
 1:
@@ -173,7 +166,7 @@ error_code:
     .word   BIONIC_EVENT_STRCPY_BUFFER_OVERFLOW
 error_message:
     .word   error_string-(1b+4)
-END(__strcpy_chk_failed)
+END(__strcpy_chk)
 
     .data
 error_string:

libc/arch-arm/krait/bionic/memcpy.S

@@ -45,7 +45,7 @@
 
 ENTRY(__memcpy_chk)
         cmp         r2, r3
-        bhi         __memcpy_chk_fail
+        bhi         .L_memcpy_chk_fail
 
         // Fall through to memcpy...
 END(__memcpy_chk)
@@ -53,19 +53,20 @@ END(__memcpy_chk)
 ENTRY(memcpy)
         pld     [r1, #64]
         stmfd   sp!, {r0, lr}
-        .cfi_def_cfa_offset 8
+        .cfi_adjust_cfa_offset 8
         .cfi_rel_offset r0, 0
         .cfi_rel_offset lr, 4
-END(memcpy)
 
-#define MEMCPY_BASE         __memcpy_base
-#define MEMCPY_BASE_ALIGNED __memcpy_base_aligned
 #include "memcpy_base.S"
 
-ENTRY_PRIVATE(__memcpy_chk_fail)
+        // Undo the cfi directives from above.
+        .cfi_adjust_cfa_offset -8
+        .cfi_restore r0
+        .cfi_restore lr
+.L_memcpy_chk_fail:
         // Preserve lr for backtrace.
         push    {lr}
-        .cfi_def_cfa_offset 4
+        .cfi_adjust_cfa_offset 4
         .cfi_rel_offset lr, 0
 
         ldr     r0, error_message
@@ -77,7 +78,7 @@ error_code:
         .word   BIONIC_EVENT_MEMCPY_BUFFER_OVERFLOW
 error_message:
         .word   error_string-(1b+4)
-END(__memcpy_chk_fail)
+END(memcpy)
 
         .data
 error_string:

libc/arch-arm/krait/bionic/memcpy_base.S

@@ -1,123 +1,191 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *  * Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- *  * Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in
- *    the documentation and/or other materials provided with the
- *    distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
- * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
- * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
+/***************************************************************************
+ Copyright (c) 2009-2013 The Linux Foundation. All rights reserved.
 
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+     * Redistributions of source code must retain the above copyright
+       notice, this list of conditions and the following disclaimer.
+     * Redistributions in binary form must reproduce the above copyright
+       notice, this list of conditions and the following disclaimer in the
+       documentation and/or other materials provided with the distribution.
+     * Neither the name of The Linux Foundation nor the names of its contributors may
+       be used to endorse or promote products derived from this software
+       without specific prior written permission.
 
-/*
- * This code assumes it is running on a processor that supports all arm v7
- * instructions, that supports neon instructions, and that has a 32 byte
- * cache line.
- */
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ POSSIBILITY OF SUCH DAMAGE.
+  ***************************************************************************/
 
-// Assumes neon instructions and a cache line size of 32 bytes.
+/* Assumes neon instructions and a cache line size of 64 bytes. */
 
-ENTRY_PRIVATE(MEMCPY_BASE)
-        .cfi_def_cfa_offset 8
-        .cfi_rel_offset r0, 0
-        .cfi_rel_offset lr, 4
+#include <machine/cpu-features.h>
+#include <machine/asm.h>
 
-        /* do we have at least 16-bytes to copy (needed for alignment below) */
+#define PLDOFFS	(10)
+#define PLDTHRESH (PLDOFFS)
+#define BBTHRESH (4096/64)
+#define PLDSIZE (64)
+
+#if (PLDOFFS < 1)
+#error Routine does not support offsets less than 1
+#endif
+
+#if (PLDTHRESH < PLDOFFS)
+#error PLD threshold must be greater than or equal to the PLD offset
+#endif
+
+	.text
+	.fpu    neon
+
+.L_memcpy_base:
+	cmp	r2, #4
+	blt	.L_neon_lt4
 	cmp	r2, #16
-        blo         5f
+	blt	.L_neon_lt16
+	cmp	r2, #32
+	blt	.L_neon_16
+	cmp	r2, #64
+	blt	.L_neon_copy_32_a
 
-        /* align destination to cache-line for the write-buffer */
-        rsb         r3, r0, #0
-        ands        r3, r3, #0xF
-        beq         2f
+	mov	r12, r2, lsr #6
+	cmp	r12, #PLDTHRESH
+	ble	.L_neon_copy_64_loop_nopld
 
-        /* copy up to 15-bytes (count in r3) */
-        sub         r2, r2, r3
-        movs        ip, r3, lsl #31
-        itt         mi
-        ldrbmi      lr, [r1], #1
-        strbmi      lr, [r0], #1
-        itttt       cs
-        ldrbcs      ip, [r1], #1
-        ldrbcs      lr, [r1], #1
-        strbcs      ip, [r0], #1
-        strbcs      lr, [r0], #1
-        movs        ip, r3, lsl #29
-        bge         1f
-        // copies 4 bytes, destination 32-bits aligned
-        vld4.8      {d0[0], d1[0], d2[0], d3[0]}, [r1]!
-        vst4.8      {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]!
-1:      bcc         2f
-        // copies 8 bytes, destination 64-bits aligned
-        vld1.8      {d0}, [r1]!
-        vst1.8      {d0}, [r0, :64]!
+	push	{r9, r10}
+	.cfi_adjust_cfa_offset 8
+	.cfi_rel_offset r9, 0
+	.cfi_rel_offset r10, 4
 
-2:      /* make sure we have at least 64 bytes to copy */
-        subs        r2, r2, #64
-        blo         2f
+	cmp	r12, #BBTHRESH
+	ble	.L_neon_prime_pump
 
-1:      /* The main loop copies 64 bytes at a time */
-        vld1.8      {d0  - d3},   [r1]!
-        vld1.8      {d4  - d7},   [r1]!
-        pld         [r1, #(32*8)]
-        subs        r2, r2, #64
-        vst1.8      {d0  - d3},   [r0, :128]!
-        vst1.8      {d4  - d7},   [r0, :128]!
-        bhs         1b
+	add	lr, r0, #0x400
+	add	r9, r1, #(PLDOFFS*PLDSIZE)
+	sub	lr, lr, r9
+	lsl	lr, lr, #21
+	lsr	lr, lr, #21
+	add	lr, lr, #(PLDOFFS*PLDSIZE)
+	cmp	r12, lr, lsr #6
+	ble	.L_neon_prime_pump
 
-2:      /* fix-up the remaining count and make sure we have >= 32 bytes left */
-        adds        r2, r2, #32
-        blo         4f
+	itt	gt
+	movgt	r9, #(PLDOFFS)
+	rsbsgt	r9, r9, lr, lsr #6
+	ble	.L_neon_prime_pump
 
-        /* Copy 32 bytes. These cache lines were already preloaded */
-        vld1.8      {d0 - d3},  [r1]!
-        sub         r2, r2, #32
-        vst1.8      {d0 - d3},  [r0, :128]!
+	add	r10, r1, lr
+	bic	r10, #0x3F
 
-4:      /* less than 32 left */
-        add         r2, r2, #32
-        tst         r2, #0x10
-        beq         5f
-        // copies 16 bytes, 128-bits aligned
-        vld1.8      {d0, d1}, [r1]!
-        vst1.8      {d0, d1}, [r0, :128]!
+	sub	r12, r12, lr, lsr #6
 
-5:      /* copy up to 15-bytes (count in r2) */
-        movs        ip, r2, lsl #29
+	cmp	r9, r12
+	itee	le
+	suble	r12, r12, r9
+	movgt	r9, r12
+	movgt	r12, #0
+
+	pld	[r1, #((PLDOFFS-1)*PLDSIZE)]
+.L_neon_copy_64_loop_outer_doublepld:
+	pld	[r1, #((PLDOFFS)*PLDSIZE)]
+	vld1.32	{q0, q1}, [r1]!
+	vld1.32	{q2, q3}, [r1]!
+	ldr	r3, [r10]
+	subs	r9, r9, #1
+	vst1.32	{q0, q1}, [r0]!
+	vst1.32	{q2, q3}, [r0]!
+	add	r10, #64
+	bne	.L_neon_copy_64_loop_outer_doublepld
+	cmp	r12, #0
+	beq	.L_neon_pop_before_nopld
+
+	cmp	r12, #(512*1024/64)
+	blt	.L_neon_copy_64_loop_outer
+
+.L_neon_copy_64_loop_ddr:
+	vld1.32	{q0, q1}, [r1]!
+	vld1.32	{q2, q3}, [r1]!
+	pld	[r10]
+	subs	r12, r12, #1
+	vst1.32	{q0, q1}, [r0]!
+	vst1.32	{q2, q3}, [r0]!
+	add	r10, #64
+	bne	.L_neon_copy_64_loop_ddr
+	b	.L_neon_pop_before_nopld
+
+.L_neon_prime_pump:
+	mov	lr, #(PLDOFFS*PLDSIZE)
+	add	r10, r1, #(PLDOFFS*PLDSIZE)
+	bic	r10, #0x3F
+	sub	r12, r12, #PLDOFFS
+	ldr	r3, [r10, #(-1*PLDSIZE)]
+
+.L_neon_copy_64_loop_outer:
+	vld1.32	{q0, q1}, [r1]!
+	vld1.32	{q2, q3}, [r1]!
+	ldr	r3, [r10]
+	subs	r12, r12, #1
+	vst1.32	{q0, q1}, [r0]!
+	vst1.32	{q2, q3}, [r0]!
+	add	r10, #64
+	bne	.L_neon_copy_64_loop_outer
+
+.L_neon_pop_before_nopld:
+	mov	r12, lr, lsr #6
+	pop	{r9, r10}
+	.cfi_adjust_cfa_offset -8
+	.cfi_restore r9
+	.cfi_restore r10
+
+.L_neon_copy_64_loop_nopld:
+	vld1.32	{q8, q9}, [r1]!
+	vld1.32	{q10, q11}, [r1]!
+	subs	r12, r12, #1
+	vst1.32	{q8, q9}, [r0]!
+	vst1.32	{q10, q11}, [r0]!
+	bne	.L_neon_copy_64_loop_nopld
+	ands	r2, r2, #0x3f
+	beq	.L_neon_exit
+
+.L_neon_copy_32_a:
+	movs	r3, r2, lsl #27
+	bcc	.L_neon_16
+	vld1.32	{q0,q1}, [r1]!
+	vst1.32	{q0,q1}, [r0]!
+
+.L_neon_16:
+	bpl	.L_neon_lt16
+	vld1.32	{q8}, [r1]!
+	vst1.32	{q8}, [r0]!
+	ands	r2, r2, #0x0f
+	beq	.L_neon_exit
+
+.L_neon_lt16:
+	movs	r3, r2, lsl #29
 	bcc	1f
 	vld1.8	{d0}, [r1]!
 	vst1.8	{d0}, [r0]!
-1:      bge         2f
+1:
+	bge	.L_neon_lt4
 	vld4.8	{d0[0], d1[0], d2[0], d3[0]}, [r1]!
 	vst4.8	{d0[0], d1[0], d2[0], d3[0]}, [r0]!
-2:      movs        ip, r2, lsl #31
-        itt         mi
-        ldrbmi      r3, [r1], #1
-        strbmi      r3, [r0], #1
-        itttt       cs
-        ldrbcs      ip, [r1], #1
-        ldrbcs      lr, [r1], #1
-        strbcs      ip, [r0], #1
-        strbcs      lr, [r0], #1
 
-        ldmfd       sp!, {r0, lr}
-        bx          lr
-END(MEMCPY_BASE)
+.L_neon_lt4:
+	movs	r2, r2, lsl #31
+	itt	cs
+	ldrhcs	r3, [r1], #2
+	strhcs	r3, [r0], #2
+	itt	mi
+	ldrbmi	r3, [r1]
+	strbmi	r3, [r0]
+
+.L_neon_exit:
+	pop	{r0, pc}

libc/arch-arm/krait/bionic/memset.S

@@ -69,10 +69,7 @@ END(bzero)
 
 /* memset() returns its first argument.  */
 ENTRY(memset)
-        stmfd       sp!, {r0}
-        .cfi_def_cfa_offset 4
-        .cfi_rel_offset r0, 0
-
+        mov         r3, r0
         vdup.8      q0, r1
 
         /* make sure we have at least 32 bytes to write */
@@ -82,7 +79,7 @@ ENTRY(memset)
 
 1:      /* The main loop writes 32 bytes at a time */
         subs        r2, r2, #32
-        vst1.8      {d0 - d3}, [r0]!
+        vst1.8      {d0 - d3}, [r3]!
         bhs         1b
 
 2:      /* less than 32 left */
@@ -91,18 +88,17 @@ ENTRY(memset)
         beq         3f
 
         // writes 16 bytes, 128-bits aligned
-        vst1.8      {d0, d1}, [r0]!
+        vst1.8      {d0, d1}, [r3]!
 3:      /* write up to 15-bytes (count in r2) */
         movs        ip, r2, lsl #29
         bcc         1f
-        vst1.8      {d0}, [r0]!
+        vst1.8      {d0}, [r3]!
 1:      bge         2f
-        vst1.32     {d0[0]}, [r0]!
+        vst1.32     {d0[0]}, [r3]!
 2:      movs        ip, r2, lsl #31
-        strbmi      r1, [r0], #1
-        strbcs      r1, [r0], #1
-        strbcs      r1, [r0], #1
-        ldmfd       sp!, {r0}
+        strbmi      r1, [r3], #1
+        strbcs      r1, [r3], #1
+        strbcs      r1, [r3], #1
         bx          lr
 END(memset)
 

libc/arch-arm/krait/krait.mk

@@ -1,3 +1,18 @@
+libc_openbsd_src_files_exclude_arm += \
+    upstream-openbsd/lib/libc/string/memmove.c \
+    upstream-openbsd/lib/libc/string/stpcpy.c \
+    upstream-openbsd/lib/libc/string/stpcpy.c \
+    upstream-openbsd/lib/libc/string/strcat.c \
+
+libc_bionic_src_files_exclude_arm += \
+    arch-arm/generic/bionic/memcpy.S \
+    arch-arm/generic/bionic/memset.S \
+    arch-arm/generic/bionic/strcmp.S \
+    arch-arm/generic/bionic/strcpy.S \
+    arch-arm/generic/bionic/strlen.c \
+    bionic/__strcat_chk.cpp \
+    bionic/__strcpy_chk.cpp \
+
 libc_bionic_src_files_arm += \
     arch-arm/krait/bionic/memcpy.S \
     arch-arm/krait/bionic/memset.S \
@@ -12,8 +27,5 @@ libc_bionic_src_files_arm += \
     arch-arm/cortex-a15/bionic/strcpy.S \
     arch-arm/cortex-a15/bionic/strlen.S \
 
-libc_bionic_src_files_arm += \
-    arch-arm/generic/bionic/memcmp.S \
-
 libc_bionic_src_files_arm += \
     arch-arm/denver/bionic/memmove.S \

libc/arch-arm/syscalls/__preadv64.S

@@ -0,0 +1,22 @@
+/* Generated by gensyscalls.py. Do not edit. */
+
+#include <private/bionic_asm.h>
+
+ENTRY(__preadv64)
+    mov     ip, sp
+    stmfd   sp!, {r4, r5, r6, r7}
+    .cfi_def_cfa_offset 16
+    .cfi_rel_offset r4, 0
+    .cfi_rel_offset r5, 4
+    .cfi_rel_offset r6, 8
+    .cfi_rel_offset r7, 12
+    ldmfd   ip, {r4, r5, r6}
+    ldr     r7, =__NR_preadv
+    swi     #0
+    ldmfd   sp!, {r4, r5, r6, r7}
+    .cfi_def_cfa_offset 0
+    cmn     r0, #(MAX_ERRNO + 1)
+    bxls    lr
+    neg     r0, r0
+    b       __set_errno_internal
+END(__preadv64)

libc/arch-arm/syscalls/__pwritev64.S

@@ -0,0 +1,22 @@
+/* Generated by gensyscalls.py. Do not edit. */
+
+#include <private/bionic_asm.h>
+
+ENTRY(__pwritev64)
+    mov     ip, sp
+    stmfd   sp!, {r4, r5, r6, r7}
+    .cfi_def_cfa_offset 16
+    .cfi_rel_offset r4, 0
+    .cfi_rel_offset r5, 4
+    .cfi_rel_offset r6, 8
+    .cfi_rel_offset r7, 12
+    ldmfd   ip, {r4, r5, r6}
+    ldr     r7, =__NR_pwritev
+    swi     #0
+    ldmfd   sp!, {r4, r5, r6, r7}
+    .cfi_def_cfa_offset 0
+    cmn     r0, #(MAX_ERRNO + 1)
+    bxls    lr
+    neg     r0, r0
+    b       __set_errno_internal
+END(__pwritev64)

libc/arch-arm/syscalls/vfork.S

@@ -1,14 +0,0 @@
-/* Generated by gensyscalls.py. Do not edit. */
-
-#include <private/bionic_asm.h>
-
-ENTRY(vfork)
-    mov     ip, r7
-    ldr     r7, =__NR_vfork
-    swi     #0
-    mov     r7, ip
-    cmn     r0, #(MAX_ERRNO + 1)
-    bxls    lr
-    neg     r0, r0
-    b       __set_errno_internal
-END(vfork)

libc/arch-arm64/arm64.mk

@@ -1,32 +1,38 @@
 # 64-bit arm.
 
 #
-# Default implementations of functions that are commonly optimized.
+# Generic arm64 optimizations, may be overriden by CPU variants.
 #
 
 libc_bionic_src_files_arm64 += \
-    bionic/__memset_chk.cpp \
-    bionic/__strcpy_chk.cpp \
-    bionic/__strcat_chk.cpp \
-    bionic/strrchr.cpp \
+    arch-arm64/generic/bionic/memchr.S \
+    arch-arm64/generic/bionic/memcmp.S \
+    arch-arm64/generic/bionic/memcpy.S \
+    arch-arm64/generic/bionic/memmove.S \
+    arch-arm64/generic/bionic/memset.S \
+    arch-arm64/generic/bionic/stpcpy.S \
+    arch-arm64/generic/bionic/strchr.S \
+    arch-arm64/generic/bionic/strcmp.S \
+    arch-arm64/generic/bionic/strcpy.S \
+    arch-arm64/generic/bionic/strlen.S \
+    arch-arm64/generic/bionic/strncmp.S \
+    arch-arm64/generic/bionic/strnlen.S \
+    arch-arm64/generic/bionic/wmemmove.S \
 
-libc_freebsd_src_files_arm64 += \
-    upstream-freebsd/lib/libc/string/wcscat.c \
-    upstream-freebsd/lib/libc/string/wcschr.c \
-    upstream-freebsd/lib/libc/string/wcscmp.c \
-    upstream-freebsd/lib/libc/string/wcscpy.c \
-    upstream-freebsd/lib/libc/string/wcslen.c \
-    upstream-freebsd/lib/libc/string/wcsrchr.c \
-    upstream-freebsd/lib/libc/string/wmemcmp.c \
+libc_bionic_src_files_exclude_arm64 += \
+    bionic/__memcpy_chk.cpp \
+    bionic/strchr.cpp \
+    bionic/strnlen.c \
 
-libc_openbsd_src_files_arm64 += \
-    upstream-openbsd/lib/libc/string/memrchr.c \
-    upstream-openbsd/lib/libc/string/stpncpy.c \
-    upstream-openbsd/lib/libc/string/strcat.c \
-    upstream-openbsd/lib/libc/string/strlcat.c \
-    upstream-openbsd/lib/libc/string/strlcpy.c \
-    upstream-openbsd/lib/libc/string/strncat.c \
-    upstream-openbsd/lib/libc/string/strncpy.c \
+libc_freebsd_src_files_exclude_arm64 += \
+    upstream-freebsd/lib/libc/string/wmemmove.c \
+
+libc_openbsd_src_files_exclude_arm64 += \
+    upstream-openbsd/lib/libc/string/memchr.c \
+    upstream-openbsd/lib/libc/string/memmove.c \
+    upstream-openbsd/lib/libc/string/stpcpy.c \
+    upstream-openbsd/lib/libc/string/strcpy.c \
+    upstream-openbsd/lib/libc/string/strncmp.c \
 
 #
 # Inherently architecture-specific code.
@@ -54,6 +60,7 @@ libc_crt_target_crtbegin_so_file_arm64 := \
 ifeq ($(strip $(TARGET_CPU_VARIANT)),)
   $(warning TARGET_ARCH is arm64, but TARGET_CPU_VARIANT is not defined)
 endif
+ifneq ($(TARGET_CPU_VARIANT),generic)
 cpu_variant_mk := $(LOCAL_PATH)/arch-arm64/$(TARGET_CPU_VARIANT)/$(TARGET_CPU_VARIANT).mk
 ifeq ($(wildcard $(cpu_variant_mk)),)
 $(error "TARGET_CPU_VARIANT not set or set to an unknown value. Possible values are generic, denver64. Use generic for devices that do not have a CPU similar to any of the supported cpu variants.")
@@ -62,3 +69,4 @@ include $(cpu_variant_mk)
 libc_common_additional_dependencies += $(cpu_variant_mk)
 
 cpu_variant_mk :=
+endif

libc/arch-arm64/bionic/setjmp.S

@@ -52,6 +52,29 @@
 #define _JB_D10_D11     (_JB_D12_D13 + 2)
 #define _JB_D8_D9       (_JB_D10_D11 + 2)
 
+#define MANGLE_REGISTERS 1
+.macro m_mangle_registers reg, sp_reg
+#if MANGLE_REGISTERS
+  eor x19, x19, \reg
+  eor x20, x20, \reg
+  eor x21, x21, \reg
+  eor x22, x22, \reg
+  eor x23, x23, \reg
+  eor x24, x24, \reg
+  eor x25, x25, \reg
+  eor x26, x26, \reg
+  eor x27, x27, \reg
+  eor x28, x28, \reg
+  eor x29, x29, \reg
+  eor x30, x30, \reg
+  eor \sp_reg, \sp_reg, \reg
+#endif
+.endm
+
+.macro m_unmangle_registers reg, sp_reg
+  m_mangle_registers \reg, sp_reg=\sp_reg
+.endm
+
 ENTRY(setjmp)
   mov w1, #1
   b sigsetjmp
@@ -64,23 +87,47 @@ END(_setjmp)
 
 // int sigsetjmp(sigjmp_buf env, int save_signal_mask);
 ENTRY(sigsetjmp)
-  // Record whether or not we're saving the signal mask.
-  str w1, [x0, #(_JB_SIGFLAG * 8)]
+  stp x0, x30, [sp, #-16]!
+  .cfi_def_cfa_offset 16
+  .cfi_rel_offset x0, 0
+  .cfi_rel_offset x30, 8
+
+  // Get the cookie and store it along with the signal flag.
+  mov x0, x1
+  bl __bionic_setjmp_cookie_get
+  mov x1, x0
+  ldr x0, [sp, #0]
+  str x1, [x0, #(_JB_SIGFLAG * 8)]
 
   // Do we need to save the signal mask?
-  cbz w1, 1f
+  tbz w1, #0, 1f
+
+  // Save the cookie for later.
+  stp x1, xzr, [sp, #-16]!
+  .cfi_adjust_cfa_offset 16
 
   // Save current signal mask.
-  stp x0, x30, [sp, #-16]!
   // The 'how' argument is ignored if new_mask is NULL.
   mov x1, #0 // NULL.
   add x2, x0, #(_JB_SIGMASK * 8) // old_mask.
   bl sigprocmask
-  ldp x0, x30, [sp], #16
+
+  ldp x1, xzr, [sp], #16
+  .cfi_adjust_cfa_offset -16
 
 1:
+  // Restore original x0 and lr.
+  ldp x0, x30, [sp], #16
+  .cfi_adjust_cfa_offset -16
+  .cfi_restore x0
+  .cfi_restore x30
+
+  // Mask off the signal flag bit.
+  bic x1, x1, #1
+
   // Save core registers.
   mov x10, sp
+  m_mangle_registers x1, sp_reg=x10
   stp x30, x10, [x0, #(_JB_X30_SP  * 8)]
   stp x28, x29, [x0, #(_JB_X28_X29 * 8)]
   stp x26, x27, [x0, #(_JB_X26_X27 * 8)]
@@ -88,6 +135,7 @@ ENTRY(sigsetjmp)
   stp x22, x23, [x0, #(_JB_X22_X23 * 8)]
   stp x20, x21, [x0, #(_JB_X20_X21 * 8)]
   str x19,      [x0, #(_JB_X19     * 8)]
+  m_unmangle_registers x1, sp_reg=x10
 
   // Save floating point registers.
   stp d14, d15, [x0, #(_JB_D14_D15 * 8)]
@@ -102,30 +150,60 @@ END(sigsetjmp)
 // void siglongjmp(sigjmp_buf env, int value);
 ENTRY(siglongjmp)
   // Do we need to restore the signal mask?
-  ldr w9, [x0, #(_JB_SIGFLAG * 8)]
-  cbz w9, 1f
+  ldr x2, [x0, #(_JB_SIGFLAG * 8)]
+  tbz w2, #0, 1f
+
+  stp x0, x30, [sp, #-16]!
+  .cfi_adjust_cfa_offset 16
+  .cfi_rel_offset x0, 0
+  .cfi_rel_offset x30, 8
 
   // Restore signal mask.
-  stp x0, x30, [sp, #-16]!
   mov x19, x1 // Save 'value'.
+
   mov x2, x0
   mov x0, #2 // SIG_SETMASK
   add x1, x2, #(_JB_SIGMASK * 8) // new_mask.
   mov x2, #0 // NULL.
   bl sigprocmask
   mov x1, x19 // Restore 'value'.
-  ldp x0, x30, [sp], #16
 
+  // Restore original x0 and lr.
+  ldp x0, x30, [sp], #16
+  .cfi_adjust_cfa_offset -16
+  .cfi_restore x0
+  .cfi_restore x30
+
+  ldr x2, [x0, #(_JB_SIGFLAG * 8)]
 1:
   // Restore core registers.
+  bic x2, x2, #1
   ldp x30, x10, [x0, #(_JB_X30_SP  * 8)]
-  mov sp, x10
   ldp x28, x29, [x0, #(_JB_X28_X29 * 8)]
   ldp x26, x27, [x0, #(_JB_X26_X27 * 8)]
   ldp x24, x25, [x0, #(_JB_X24_X25 * 8)]
   ldp x22, x23, [x0, #(_JB_X22_X23 * 8)]
   ldp x20, x21, [x0, #(_JB_X20_X21 * 8)]
   ldr x19,      [x0, #(_JB_X19     * 8)]
+  m_unmangle_registers x2, sp_reg=x10
+  mov sp, x10
+
+  stp x0, x1, [sp, #-16]!
+  .cfi_adjust_cfa_offset 16
+  .cfi_rel_offset x0, 0
+  .cfi_rel_offset x1, 8
+  stp x30, xzr, [sp, #-16]!
+  .cfi_adjust_cfa_offset 16
+  .cfi_rel_offset x30, 0
+  ldr x0, [x0, #(_JB_SIGFLAG * 8)]
+  bl __bionic_setjmp_cookie_check
+  ldp x30, xzr, [sp], #16
+  .cfi_adjust_cfa_offset -16
+  .cfi_restore x30
+  ldp x0, x1, [sp], #16
+  .cfi_adjust_cfa_offset -16
+  .cfi_restore x0
+  .cfi_restore x1
 
   // Restore floating point registers.
   ldp d14, d15, [x0, #(_JB_D14_D15 * 8)]
@@ -133,13 +211,6 @@ ENTRY(siglongjmp)
   ldp d10, d11, [x0, #(_JB_D10_D11 * 8)]
   ldp d8,  d9,  [x0, #(_JB_D8_D9   * 8)]
 
-  // Validate sp (sp mod 16 = 0) and lr (lr mod 4 = 0).
-  tst x30, #3
-  b.ne longjmperror
-  mov x10, sp
-  tst x10, #15
-  b.ne longjmperror
-
   // Set return value.
   cmp w1, wzr
   csinc w0, w1, wzr, ne

libc/arch-arm64/bionic/vfork.S

@@ -31,6 +31,11 @@
 #include <linux/sched.h>
 
 ENTRY(vfork)
+    // __get_tls()[TLS_SLOT_THREAD_ID]->cached_pid_ = 0
+    mrs     x0, tpidr_el0
+    ldr     x0, [x0, #8]
+    str     wzr, [x0, #20]
+
     mov     x0, #(CLONE_VM | CLONE_VFORK | SIGCHLD)
     mov     x1, xzr
     mov     x2, xzr

libc/arch-arm64/cortex-a53/cortex-a53.mk

@@ -1 +0,0 @@
-include bionic/libc/arch-arm64/generic/generic.mk

libc/arch-arm64/denver64/denver64.mk

@@ -1,14 +1,7 @@
 libc_bionic_src_files_arm64 += \
-    arch-arm64/generic/bionic/memchr.S \
-    arch-arm64/generic/bionic/memcmp.S \
     arch-arm64/denver64/bionic/memcpy.S \
-    arch-arm64/generic/bionic/memmove.S \
     arch-arm64/denver64/bionic/memset.S \
-    arch-arm64/generic/bionic/stpcpy.S \
-    arch-arm64/generic/bionic/strchr.S \
-    arch-arm64/generic/bionic/strcmp.S \
-    arch-arm64/generic/bionic/strcpy.S \
-    arch-arm64/generic/bionic/strlen.S \
-    arch-arm64/generic/bionic/strncmp.S \
-    arch-arm64/generic/bionic/strnlen.S \
-    arch-arm64/generic/bionic/wmemmove.S
+
+libc_bionic_src_files_exclude_arm64 += \
+    arch-arm64/generic/bionic/memcpy.S \
+    arch-arm64/generic/bionic/memset.S \

libc/arch-arm64/generic/bionic/memmove.S

@@ -35,10 +35,6 @@
 #include <private/bionic_asm.h>
 
 /* Parameters and result.  */
-#ifdef BCOPY
-#define origdstin	x1
-#define origsrc	x0
-#endif
 #define dstin	x0
 #define src	x1
 #define count	x2
@@ -59,13 +55,7 @@
 #define D_l	x13
 #define D_h	x14
 
-#ifdef BCOPY
-ENTRY(bcopy)
-	/* Swap src and dst so that a branch to memcpy doesn't cause issues. */
-	mov	tmp1, origsrc
-	mov	origsrc, origdstin
-	mov	origdstin, tmp1
-#elif defined(WMEMMOVE)
+#if defined(WMEMMOVE)
 ENTRY(wmemmove)
 	lsl	count, count, #2
 #else
@@ -332,9 +322,7 @@ ENTRY(memmove)
 	tst	count, #0x3f
 	b.ne	.Ltail63down
 	ret
-#ifdef BCOPY
-END(bcopy)
-#elif defined(WMEMMOVE)
+#if defined(WMEMMOVE)
 END(wmemmove)
 #else
 END(memmove)

libc/arch-arm64/generic/generic.mk

@@ -1,14 +0,0 @@
-libc_bionic_src_files_arm64 += \
-    arch-arm64/generic/bionic/memchr.S \
-    arch-arm64/generic/bionic/memcmp.S \
-    arch-arm64/generic/bionic/memcpy.S \
-    arch-arm64/generic/bionic/memmove.S \
-    arch-arm64/generic/bionic/memset.S \
-    arch-arm64/generic/bionic/stpcpy.S \
-    arch-arm64/generic/bionic/strchr.S \
-    arch-arm64/generic/bionic/strcmp.S \
-    arch-arm64/generic/bionic/strcpy.S \
-    arch-arm64/generic/bionic/strlen.S \
-    arch-arm64/generic/bionic/strncmp.S \
-    arch-arm64/generic/bionic/strnlen.S \
-    arch-arm64/generic/bionic/wmemmove.S

libc/arch-arm64/syscalls/preadv.S

@@ -0,0 +1,16 @@
+/* Generated by gensyscalls.py. Do not edit. */
+
+#include <private/bionic_asm.h>
+
+ENTRY(preadv)
+    mov     x8, __NR_preadv
+    svc     #0
+
+    cmn     x0, #(MAX_ERRNO + 1)
+    cneg    x0, x0, hi
+    b.hi    __set_errno_internal
+
+    ret
+END(preadv)
+
+ALIAS_SYMBOL(preadv64, preadv)

libc/arch-arm64/syscalls/pwritev.S

@@ -0,0 +1,16 @@
+/* Generated by gensyscalls.py. Do not edit. */
+
+#include <private/bionic_asm.h>
+
+ENTRY(pwritev)
+    mov     x8, __NR_pwritev
+    svc     #0
+
+    cmn     x0, #(MAX_ERRNO + 1)
+    cneg    x0, x0, hi
+    b.hi    __set_errno_internal
+
+    ret
+END(pwritev)
+
+ALIAS_SYMBOL(pwritev64, pwritev)

libc/arch-mips/bionic/setjmp.S

@@ -132,10 +132,9 @@
 /*     	field:		byte offset:	size:						*/
 /*     	dynam filler	(0*4)		   0-4 bytes of rounddown filler, DON'T TOUCH!!
 						often overlays user storage!!		*/
-#define	SC_MAGIC_OFFSET	(1*4)		/* 4 bytes, identify jmpbuf, first actual field */
-#define	SC_FLAG_OFFSET	(2*4)		/* 4 bytes, savesigs flag */
-#define	SC_FPSR_OFFSET	(3*4)		/* 4 bytes, floating point control/status reg */
+#define	SC_FPSR_OFFSET	(1*4)		/* 4 bytes, floating point control/status reg */
 /* following fields are 8-byte aligned */
+#define	SC_FLAG_OFFSET	(2*4)		/* 8 bytes, cookie and savesigs flag, first actual field  */
 #define	SC_MASK_OFFSET	(4*4)		/* 16 bytes, mips32/mips64 version of sigset_t */
 #define	SC_SPARE_OFFSET	(8*4)		/* 8 bytes, reserved for future uses */
 
@@ -166,6 +165,16 @@
 #error _JBLEN is too small
 #endif
 
+.macro m_mangle_reg_and_store reg, cookie, temp, offset
+	xor	\temp, \reg, \cookie
+	REG_S	\temp, \offset
+.endm
+
+.macro m_unmangle_reg_and_load reg, cookie, temp, offset
+	REG_L	\temp, \offset
+	xor	\reg, \temp, \cookie
+.endm
+
 /*
  *
  *  GPOFF and FRAMESIZE must be the same for all setjmp/longjmp routines
@@ -190,36 +199,46 @@ setjmp_common:
 	li	t0, ~7
 	and	a0, t0				# round jmpbuf addr DOWN to 8-byte boundary
 #endif
-	sw	a1, SC_FLAG_OFFSET(a0)		# save savesigs flag
-	beqz	a1, 1f				# do saving of signal mask?
-
 	REG_S	ra, RAOFF(sp)			# spill state
 	REG_S	a0, A0OFF(sp)
+
+	# get the cookie and store it along with the signal flag.
+	move	a0, a1
+	jal	__bionic_setjmp_cookie_get
+	REG_L	a0, A0OFF(sp)
+
+	REG_S	v0, SC_FLAG_OFFSET(a0)		# save cookie and savesigs flag
+	andi	t0, v0, 1			# extract savesigs flag
+
+	beqz	t0, 1f				# do saving of signal mask?
+
 	# call sigprocmask(int how ignored, sigset_t* null, sigset_t* SC_MASK(a0)):
 	LA	a2, SC_MASK_OFFSET(a0)		# gets current signal mask
 	li	a0, 0				# how; ignored when new mask is null
 	li	a1, 0				# null new mask
 	jal	sigprocmask			# get current signal mask
 	REG_L	a0, A0OFF(sp)
-	REG_L	ra, RAOFF(sp)
 1:
-	li	v0, 0xACEDBADE			# sigcontext magic number
-	sw	v0, SC_MAGIC_OFFSET(a0)
+	REG_L	gp, GPOFF(sp)			# restore spills
+	REG_L	ra, RAOFF(sp)
+	REG_L	t0, SC_FLAG_OFFSET(a0)		# move cookie to temp reg
+
 	# callee-saved long-sized regs:
-	REG_S	ra, SC_REGS+0*REGSZ(a0)
-	REG_S	s0, SC_REGS+1*REGSZ(a0)
-	REG_S	s1, SC_REGS+2*REGSZ(a0)
-	REG_S	s2, SC_REGS+3*REGSZ(a0)
-	REG_S	s3, SC_REGS+4*REGSZ(a0)
-	REG_S	s4, SC_REGS+5*REGSZ(a0)
-	REG_S	s5, SC_REGS+6*REGSZ(a0)
-	REG_S	s6, SC_REGS+7*REGSZ(a0)
-	REG_S	s7, SC_REGS+8*REGSZ(a0)
-	REG_S	s8, SC_REGS+9*REGSZ(a0)
-	REG_L	v0, GPOFF(sp)
-	REG_S	v0, SC_REGS+10*REGSZ(a0)	# save gp
-	PTR_ADDU v0, sp, FRAMESZ
-	REG_S	v0, SC_REGS+11*REGSZ(a0)	# save orig sp
+	PTR_ADDU v1, sp, FRAMESZ		# save orig sp
+
+	# m_mangle_reg_and_store reg, cookie, temp, offset
+	m_mangle_reg_and_store	ra, t0, t1, SC_REGS+0*REGSZ(a0)
+	m_mangle_reg_and_store	s0, t0, t2, SC_REGS+1*REGSZ(a0)
+	m_mangle_reg_and_store	s1, t0, t3, SC_REGS+2*REGSZ(a0)
+	m_mangle_reg_and_store	s2, t0, t1, SC_REGS+3*REGSZ(a0)
+	m_mangle_reg_and_store	s3, t0, t2, SC_REGS+4*REGSZ(a0)
+	m_mangle_reg_and_store	s4, t0, t3, SC_REGS+5*REGSZ(a0)
+	m_mangle_reg_and_store	s5, t0, t1, SC_REGS+6*REGSZ(a0)
+	m_mangle_reg_and_store	s6, t0, t2, SC_REGS+7*REGSZ(a0)
+	m_mangle_reg_and_store	s7, t0, t3, SC_REGS+8*REGSZ(a0)
+	m_mangle_reg_and_store	s8, t0, t1, SC_REGS+9*REGSZ(a0)
+	m_mangle_reg_and_store	gp, t0, t2, SC_REGS+10*REGSZ(a0)
+	m_mangle_reg_and_store	v1, t0, t3, SC_REGS+11*REGSZ(a0)
 
 	cfc1	v0, $31
 
@@ -288,36 +307,41 @@ NON_LEAF(siglongjmp, FRAMESZ, ra)
 	li	t0, ~7
 	and	a0, t0				# round jmpbuf addr DOWN to 8-byte boundary
 #endif
-	lw	v0, SC_MAGIC_OFFSET(a0)
-	li	t0, 0xACEDBADE
-	bne	v0, t0, longjmp_botch		# jump if error
 
-	lw	t0, SC_FLAG_OFFSET(a0)		# get savesigs flag
+	move	s1, a1				# temp spill
+	move	s0, a0
+
+	# extract savesigs flag
+	REG_L	s2, SC_FLAG_OFFSET(s0)
+	andi	t0, s2, 1
 	beqz	t0, 1f				# restore signal mask?
 
-	REG_S	a1, A1OFF(sp)			# temp spill
-	REG_S	a0, A0OFF(sp)
 	# call sigprocmask(int how SIG_SETMASK, sigset_t* SC_MASK(a0), sigset_t* null):
-	LA	a1, SC_MASK_OFFSET(a0)		# signals being restored
+	LA	a1, SC_MASK_OFFSET(s0)		# signals being restored
 	li	a0, 3				# mips SIG_SETMASK
 	li	a2, 0				# null
 	jal	sigprocmask			# restore signal mask
-	REG_L	a0, A0OFF(sp)
-	REG_L	a1, A1OFF(sp)
 1:
+	move	t0, s2				# get cookie to temp reg
+	move	a1, s1
+	move	a0, s0
+
 	# callee-saved long-sized regs:
-	REG_L	ra, SC_REGS+0*REGSZ(a0)
-	REG_L	s0, SC_REGS+1*REGSZ(a0)
-	REG_L	s1, SC_REGS+2*REGSZ(a0)
-	REG_L	s2, SC_REGS+3*REGSZ(a0)
-	REG_L	s3, SC_REGS+4*REGSZ(a0)
-	REG_L	s4, SC_REGS+5*REGSZ(a0)
-	REG_L	s5, SC_REGS+6*REGSZ(a0)
-	REG_L	s6, SC_REGS+7*REGSZ(a0)
-	REG_L	s7, SC_REGS+8*REGSZ(a0)
-	REG_L	s8, SC_REGS+9*REGSZ(a0)
-	REG_L	gp, SC_REGS+10*REGSZ(a0)
-	REG_L	sp, SC_REGS+11*REGSZ(a0)
+
+	# m_unmangle_reg_and_load reg, cookie, temp, offset
+	# don't restore gp yet, old value is needed for cookie_check call
+	m_unmangle_reg_and_load ra, t0, t1, SC_REGS+0*REGSZ(a0)
+	m_unmangle_reg_and_load s0, t0, t2, SC_REGS+1*REGSZ(a0)
+	m_unmangle_reg_and_load s1, t0, t3, SC_REGS+2*REGSZ(a0)
+	m_unmangle_reg_and_load s2, t0, t1, SC_REGS+3*REGSZ(a0)
+	m_unmangle_reg_and_load s3, t0, t2, SC_REGS+4*REGSZ(a0)
+	m_unmangle_reg_and_load s4, t0, t3, SC_REGS+5*REGSZ(a0)
+	m_unmangle_reg_and_load s5, t0, t1, SC_REGS+6*REGSZ(a0)
+	m_unmangle_reg_and_load s6, t0, t2, SC_REGS+7*REGSZ(a0)
+	m_unmangle_reg_and_load s7, t0, t3, SC_REGS+8*REGSZ(a0)
+	m_unmangle_reg_and_load s8, t0, t1, SC_REGS+9*REGSZ(a0)
+	m_unmangle_reg_and_load v1, t0, t2, SC_REGS+10*REGSZ(a0)
+	m_unmangle_reg_and_load sp, t0, t3, SC_REGS+11*REGSZ(a0)
 
 	lw	v0, SC_FPSR_OFFSET(a0)
 	ctc1	v0, $31			# restore old fr mode before fp values
@@ -341,15 +365,22 @@ NON_LEAF(siglongjmp, FRAMESZ, ra)
 	l.d	$f28, SC_FPREGS+4*REGSZ_FP(a0)
 	l.d	$f30, SC_FPREGS+5*REGSZ_FP(a0)
 #endif
-	bne	a1, zero, 1f
-	li	a1, 1			# never return 0!
-1:
-	move	v0, a1
-	j	ra			# return to setjmp call site
 
-longjmp_botch:
-	jal	longjmperror
-	jal	abort
+	# check cookie
+	PTR_SUBU sp, FRAMESZ
+	REG_S	v1, GPOFF(sp)
+	REG_S	ra, RAOFF(sp)
+	REG_S	a1, A1OFF(sp)
+	move	a0, t0
+	jal	__bionic_setjmp_cookie_check
+	REG_L	gp, GPOFF(sp)
+	REG_L	ra, RAOFF(sp)
+	REG_L	a1, A1OFF(sp)
+	PTR_ADDU sp, FRAMESZ
+
+	sltiu	t0, a1, 1		# never return 0!
+	xor	v0, a1, t0
+	j	ra			# return to setjmp call site
 END(siglongjmp)
 
 ALIAS_SYMBOL(longjmp, siglongjmp)

libc/arch-mips/bionic/vfork.S

@@ -37,6 +37,14 @@ ENTRY(vfork)
 	.set	noreorder
 	.cpload	t9
 
+	// __get_tls()[TLS_SLOT_THREAD_ID]->cached_pid_ = 0
+	.set	push
+	.set	mips32r2
+	rdhwr	v0, $29			// v0 = tls; kernel trap on mips32r1
+	.set	pop
+	lw	v0, REGSZ*1(v0)		// v0 = v0[TLS_SLOT_THREAD_ID ie 1]
+	sw	$0, REGSZ*2+4(v0)	// v0->cached_pid_ = 0
+
 	li	a0, (CLONE_VM | CLONE_VFORK | SIGCHLD)
 	li	a1, 0
 	li	a2, 0

libc/arch-mips/mips.mk

@@ -1,45 +1,10 @@
 # 32-bit mips.
 
-#
-# Default implementations of functions that are commonly optimized.
-#
-
 libc_bionic_src_files_mips += \
     arch-mips/string/memcmp.c \
     arch-mips/string/memcpy.S \
     arch-mips/string/memset.S \
     arch-mips/string/strcmp.S \
-    bionic/__memcpy_chk.cpp \
-    bionic/__memset_chk.cpp \
-    bionic/__strcpy_chk.cpp \
-    bionic/__strcat_chk.cpp \
-    bionic/strchr.cpp \
-    bionic/strnlen.c \
-    bionic/strrchr.cpp \
-
-libc_freebsd_src_files_mips += \
-    upstream-freebsd/lib/libc/string/wcscat.c \
-    upstream-freebsd/lib/libc/string/wcschr.c \
-    upstream-freebsd/lib/libc/string/wcscmp.c \
-    upstream-freebsd/lib/libc/string/wcscpy.c \
-    upstream-freebsd/lib/libc/string/wcslen.c \
-    upstream-freebsd/lib/libc/string/wcsrchr.c \
-    upstream-freebsd/lib/libc/string/wmemcmp.c \
-    upstream-freebsd/lib/libc/string/wmemmove.c \
-
-libc_openbsd_src_files_mips += \
-    upstream-openbsd/lib/libc/string/memchr.c \
-    upstream-openbsd/lib/libc/string/memmove.c \
-    upstream-openbsd/lib/libc/string/memrchr.c \
-    upstream-openbsd/lib/libc/string/stpcpy.c \
-    upstream-openbsd/lib/libc/string/stpncpy.c \
-    upstream-openbsd/lib/libc/string/strcat.c \
-    upstream-openbsd/lib/libc/string/strcpy.c \
-    upstream-openbsd/lib/libc/string/strlcat.c \
-    upstream-openbsd/lib/libc/string/strlcpy.c \
-    upstream-openbsd/lib/libc/string/strncat.c \
-    upstream-openbsd/lib/libc/string/strncmp.c \
-    upstream-openbsd/lib/libc/string/strncpy.c \
 
 #
 # Inherently architecture-specific code.

libc/arch-mips/syscalls/__preadv64.S

@@ -0,0 +1,19 @@
+/* Generated by gensyscalls.py. Do not edit. */
+
+#include <private/bionic_asm.h>
+
+ENTRY(__preadv64)
+    .set noreorder
+    .cpload t9
+    li v0, __NR_preadv
+    syscall
+    bnez a3, 1f
+    move a0, v0
+    j ra
+    nop
+1:
+    la t9,__set_errno_internal
+    j t9
+    nop
+    .set reorder
+END(__preadv64)

libc/arch-mips/syscalls/__pwritev64.S

@@ -0,0 +1,19 @@
+/* Generated by gensyscalls.py. Do not edit. */
+
+#include <private/bionic_asm.h>
+
+ENTRY(__pwritev64)
+    .set noreorder
+    .cpload t9
+    li v0, __NR_pwritev
+    syscall
+    bnez a3, 1f
+    move a0, v0
+    j ra
+    nop
+1:
+    la t9,__set_errno_internal
+    j t9
+    nop
+    .set reorder
+END(__pwritev64)

libc/arch-mips64/bionic/vfork.S

@@ -46,6 +46,12 @@ LEAF(vfork,FRAMESZ)
 	PTR_SUBU sp, FRAMESZ
 #endif
 	SETUP_GP64(a5, vfork)
+
+	// __get_tls()[TLS_SLOT_THREAD_ID]->cached_pid_ = 0
+	rdhwr	v0, $29			// v0 = tls
+	REG_L	v0, REGSZ*1(v0)		// v0 = v0[TLS_SLOT_THREAD_ID ie 1]
+	sw	$0, REGSZ*2+4(v0)	// v0->cached_pid_ = 0
+
 	LI	a0, (CLONE_VM | CLONE_VFORK | SIGCHLD)
 	move	a1, $0
 	move	a2, $0

libc/arch-mips64/mips64.mk

@@ -1,46 +1,11 @@
 # 64-bit mips.
 
-#
-# Default implementations of functions that are commonly optimized.
-#
-
 libc_bionic_src_files_mips64 += \
     arch-mips/string/memcmp.c \
     arch-mips/string/memcpy.S \
     arch-mips/string/memset.S \
     arch-mips/string/strcmp.S \
     arch-mips/string/strlen.c \
-    bionic/__memcpy_chk.cpp \
-    bionic/__memset_chk.cpp \
-    bionic/__strcpy_chk.cpp \
-    bionic/__strcat_chk.cpp \
-    bionic/strchr.cpp \
-    bionic/strnlen.c \
-    bionic/strrchr.cpp \
-
-libc_freebsd_src_files_mips64 += \
-    upstream-freebsd/lib/libc/string/wcscat.c \
-    upstream-freebsd/lib/libc/string/wcschr.c \
-    upstream-freebsd/lib/libc/string/wcscmp.c \
-    upstream-freebsd/lib/libc/string/wcscpy.c \
-    upstream-freebsd/lib/libc/string/wcslen.c \
-    upstream-freebsd/lib/libc/string/wcsrchr.c \
-    upstream-freebsd/lib/libc/string/wmemcmp.c \
-    upstream-freebsd/lib/libc/string/wmemmove.c \
-
-libc_openbsd_src_files_mips64 += \
-    upstream-openbsd/lib/libc/string/memchr.c \
-    upstream-openbsd/lib/libc/string/memmove.c \
-    upstream-openbsd/lib/libc/string/memrchr.c \
-    upstream-openbsd/lib/libc/string/stpcpy.c \
-    upstream-openbsd/lib/libc/string/stpncpy.c \
-    upstream-openbsd/lib/libc/string/strcat.c \
-    upstream-openbsd/lib/libc/string/strcpy.c \
-    upstream-openbsd/lib/libc/string/strlcat.c \
-    upstream-openbsd/lib/libc/string/strlcpy.c \
-    upstream-openbsd/lib/libc/string/strncat.c \
-    upstream-openbsd/lib/libc/string/strncmp.c \
-    upstream-openbsd/lib/libc/string/strncpy.c \
 
 #
 # Inherently architecture-specific code.

libc/arch-mips64/syscalls/preadv.S

@@ -0,0 +1,27 @@
+/* Generated by gensyscalls.py. Do not edit. */
+
+#include <private/bionic_asm.h>
+
+ENTRY(preadv)
+    .set push
+    .set noreorder
+    li v0, __NR_preadv
+    syscall
+    bnez a3, 1f
+    move a0, v0
+    j ra
+    nop
+1:
+    move t0, ra
+    bal     2f
+    nop
+2:
+    .cpsetup ra, t1, 2b
+    LA t9,__set_errno_internal
+    .cpreturn
+    j t9
+    move ra, t0
+    .set pop
+END(preadv)
+
+ALIAS_SYMBOL(preadv64, preadv)

libc/arch-mips64/syscalls/pwritev.S

@@ -0,0 +1,27 @@
+/* Generated by gensyscalls.py. Do not edit. */
+
+#include <private/bionic_asm.h>
+
+ENTRY(pwritev)
+    .set push
+    .set noreorder
+    li v0, __NR_pwritev
+    syscall
+    bnez a3, 1f
+    move a0, v0
+    j ra
+    nop
+1:
+    move t0, ra
+    bal     2f
+    nop
+2:
+    .cpsetup ra, t1, 2b
+    LA t9,__set_errno_internal
+    .cpreturn
+    j t9
+    move ra, t0
+    .set pop
+END(pwritev)
+
+ALIAS_SYMBOL(pwritev64, pwritev)

libc/arch-x86/atom/atom.mk

@@ -1,32 +1,27 @@
 libc_bionic_src_files_x86 += \
     arch-x86/atom/string/sse2-bzero-atom.S \
-    arch-x86/atom/string/sse2-memchr-atom.S \
-    arch-x86/atom/string/sse2-memrchr-atom.S \
     arch-x86/atom/string/sse2-memset-atom.S \
-    arch-x86/atom/string/sse2-strchr-atom.S \
     arch-x86/atom/string/sse2-strlen-atom.S \
-    arch-x86/atom/string/sse2-strnlen-atom.S \
-    arch-x86/atom/string/sse2-strrchr-atom.S \
-    arch-x86/atom/string/sse2-wcschr-atom.S \
-    arch-x86/atom/string/sse2-wcsrchr-atom.S \
-    arch-x86/atom/string/sse2-wcslen-atom.S \
-    arch-x86/atom/string/sse2-wcscmp-atom.S \
     arch-x86/atom/string/ssse3-bcopy-atom.S \
     arch-x86/atom/string/ssse3-memcmp-atom.S \
     arch-x86/atom/string/ssse3-memcpy-atom.S \
     arch-x86/atom/string/ssse3-memmove-atom.S \
-    arch-x86/atom/string/ssse3-strcat-atom.S \
-    arch-x86/atom/string/ssse3-strcmp-atom.S \
     arch-x86/atom/string/ssse3-strcpy-atom.S \
-    arch-x86/atom/string/ssse3-strlcat-atom.S \
-    arch-x86/atom/string/ssse3-strlcpy-atom.S \
-    arch-x86/atom/string/ssse3-strncat-atom.S \
-    arch-x86/atom/string/ssse3-strncmp-atom.S \
     arch-x86/atom/string/ssse3-strncpy-atom.S \
-    arch-x86/atom/string/ssse3-wcscat-atom.S \
-    arch-x86/atom/string/ssse3-wcscpy-atom.S \
     arch-x86/atom/string/ssse3-wmemcmp-atom.S
 
-libc_bionic_src_files_x86 += \
-    arch-x86/silvermont/string/sse2-stpcpy-slm.S \
-    arch-x86/silvermont/string/sse2-stpncpy-slm.S
+libc_bionic_src_files_exclude_x86 += \
+    arch-x86/generic/string/memcmp.S \
+
+libc_bionic_src_files_exclude_x86 += \
+    arch-x86/silvermont/string/sse2-bcopy-slm.S \
+    arch-x86/silvermont/string/sse2-bzero-slm.S \
+    arch-x86/silvermont/string/sse2-memcpy-slm.S \
+    arch-x86/silvermont/string/sse2-memmove-slm.S \
+    arch-x86/silvermont/string/sse2-memset-slm.S \
+    arch-x86/silvermont/string/sse2-strcpy-slm.S \
+    arch-x86/silvermont/string/sse2-strlen-slm.S \
+    arch-x86/silvermont/string/sse2-strncpy-slm.S \
+
+libc_freebsd_src_files_exclude_x86 += \
+    upstream-freebsd/lib/libc/string/wmemcmp.c \

libc/arch-x86/bionic/setjmp.S

@@ -41,30 +41,49 @@
 #define _JB_SIGMASK 6
 #define _JB_SIGFLAG 7
 
+.macro m_mangle_registers reg
+  xorl \reg,%edx
+  xorl \reg,%ebx
+  xorl \reg,%esp
+  xorl \reg,%ebp
+  xorl \reg,%esi
+  xorl \reg,%edi
+.endm
+
+.macro m_unmangle_registers reg
+  m_mangle_registers \reg
+.endm
+
 ENTRY(setjmp)
   movl 4(%esp),%ecx
-  movl $1,(_JB_SIGFLAG * 4)(%ecx)
-  jmp .L_sigsetjmp_signal_mask
+  mov $1,%eax
+  jmp .L_sigsetjmp
 END(setjmp)
 
 ENTRY(_setjmp)
   movl 4(%esp),%ecx
-  movl $0,(_JB_SIGFLAG * 4)(%ecx)
-  jmp .L_sigsetjmp_no_signal_mask
+  movl $0,%eax
+  jmp .L_sigsetjmp
 END(_setjmp)
 
 ENTRY(sigsetjmp)
   movl 4(%esp),%ecx
   movl 8(%esp),%eax
 
-  // Record whether or not the signal mask is valid.
+.L_sigsetjmp:
+  PIC_PROLOGUE
+  pushl %eax
+  call PIC_PLT(__bionic_setjmp_cookie_get)
+  addl $4,%esp
+  PIC_EPILOGUE
+
+  // Record the setjmp cookie and whether or not we're saving the signal mask.
   movl %eax,(_JB_SIGFLAG * 4)(%ecx)
 
   // Do we need to save the signal mask?
-  testl %eax,%eax
+  testl $1,%eax
   jz 1f
 
-.L_sigsetjmp_signal_mask:
   // Get the current signal mask.
   PIC_PROLOGUE
   pushl $0
@@ -76,16 +95,21 @@ ENTRY(sigsetjmp)
   movl 4(%esp),%ecx
   movl %eax,(_JB_SIGMASK * 4)(%ecx)
 
-.L_sigsetjmp_no_signal_mask:
 1:
+  // Fetch the setjmp cookie and clear the signal flag bit.
+  movl (_JB_SIGFLAG * 4)(%ecx),%eax
+  andl $-2,%eax
+
   // Save the callee-save registers.
   movl 0(%esp),%edx
+  m_mangle_registers %eax
   movl %edx,(_JB_EDX * 4)(%ecx)
   movl %ebx,(_JB_EBX * 4)(%ecx)
   movl %esp,(_JB_ESP * 4)(%ecx)
   movl %ebp,(_JB_EBP * 4)(%ecx)
   movl %esi,(_JB_ESI * 4)(%ecx)
   movl %edi,(_JB_EDI * 4)(%ecx)
+  m_unmangle_registers %eax
 
   xorl %eax,%eax
   ret
@@ -94,7 +118,8 @@ END(sigsetjmp)
 ENTRY(siglongjmp)
   // Do we have a signal mask to restore?
   movl 4(%esp),%edx
-  cmpl $0,(_JB_SIGFLAG * 4)(%edx)
+  movl (_JB_SIGFLAG * 4)(%edx), %eax
+  testl $1,%eax
   jz 1f
 
   // Restore the signal mask.
@@ -108,12 +133,31 @@ ENTRY(siglongjmp)
   // Restore the callee-save registers.
   movl 4(%esp),%edx
   movl 8(%esp),%eax
-  movl (_JB_EDX * 4)(%edx),%ecx
-  movl (_JB_EBX * 4)(%edx),%ebx
-  movl (_JB_ESP * 4)(%edx),%esp
-  movl (_JB_EBP * 4)(%edx),%ebp
-  movl (_JB_ESI * 4)(%edx),%esi
-  movl (_JB_EDI * 4)(%edx),%edi
+
+  movl (_JB_SIGFLAG * 4)(%edx),%ecx
+  andl $-2,%ecx
+
+  movl %ecx,%ebx
+  movl %ecx,%esp
+  movl %ecx,%ebp
+  movl %ecx,%esi
+  movl %ecx,%edi
+  xorl (_JB_EDX * 4)(%edx),%ecx
+  xorl (_JB_EBX * 4)(%edx),%ebx
+  xorl (_JB_ESP * 4)(%edx),%esp
+  xorl (_JB_EBP * 4)(%edx),%ebp
+  xorl (_JB_ESI * 4)(%edx),%esi
+  xorl (_JB_EDI * 4)(%edx),%edi
+
+  PIC_PROLOGUE
+  pushl %eax
+  pushl %ecx
+  pushl (_JB_SIGFLAG * 4)(%edx)
+  call PIC_PLT(__bionic_setjmp_cookie_check)
+  addl $4,%esp
+  popl %ecx
+  popl %eax
+  PIC_EPILOGUE
 
   testl %eax,%eax
   jnz 2f

libc/arch-x86/bionic/vfork.S

@@ -34,6 +34,12 @@ ENTRY(vfork)
   popl    %ecx  // Grab the return address.
   .cfi_adjust_cfa_offset 4
   .cfi_rel_offset ecx, 0
+
+  // __get_tls()[TLS_SLOT_THREAD_ID]->cached_pid_ = 0
+  movl    %gs:0, %eax
+  movl    4(%eax), %eax
+  movl    $0, 12(%eax)
+
   movl    $__NR_vfork, %eax
   int     $0x80
   cmpl    $-MAX_ERRNO, %eax

libc/arch-x86/generic/generic.mk

@@ -1,52 +0,0 @@
-libc_bionic_src_files_x86 += \
-    arch-x86/atom/string/sse2-memchr-atom.S \
-    arch-x86/atom/string/sse2-memrchr-atom.S \
-    arch-x86/atom/string/sse2-strchr-atom.S \
-    arch-x86/atom/string/sse2-strnlen-atom.S \
-    arch-x86/atom/string/sse2-strrchr-atom.S \
-    arch-x86/atom/string/sse2-wcschr-atom.S \
-    arch-x86/atom/string/sse2-wcsrchr-atom.S \
-    arch-x86/atom/string/sse2-wcslen-atom.S \
-    arch-x86/atom/string/sse2-wcscmp-atom.S \
-    arch-x86/silvermont/string/sse2-bcopy-slm.S \
-    arch-x86/silvermont/string/sse2-bzero-slm.S \
-    arch-x86/silvermont/string/sse2-memcpy-slm.S \
-    arch-x86/silvermont/string/sse2-memmove-slm.S \
-    arch-x86/silvermont/string/sse2-memset-slm.S \
-    arch-x86/silvermont/string/sse2-stpcpy-slm.S \
-    arch-x86/silvermont/string/sse2-stpncpy-slm.S \
-    arch-x86/silvermont/string/sse2-strcpy-slm.S \
-    arch-x86/silvermont/string/sse2-strlen-slm.S \
-    arch-x86/silvermont/string/sse2-strncpy-slm.S
-
-ifeq ($(ARCH_X86_HAVE_SSSE3),true)
-libc_bionic_src_files_x86 += \
-    arch-x86/atom/string/ssse3-strncat-atom.S \
-    arch-x86/atom/string/ssse3-strlcat-atom.S \
-    arch-x86/atom/string/ssse3-strlcpy-atom.S \
-    arch-x86/atom/string/ssse3-strcmp-atom.S \
-    arch-x86/atom/string/ssse3-strncmp-atom.S \
-    arch-x86/atom/string/ssse3-strcat-atom.S \
-    arch-x86/atom/string/ssse3-wcscat-atom.S \
-    arch-x86/atom/string/ssse3-wcscpy-atom.S
-else
-libc_bionic_src_files_x86 += \
-    arch-x86/generic/string/strcmp.S \
-    arch-x86/generic/string/strncmp.S \
-    arch-x86/generic/string/strcat.S \
-    upstream-freebsd/lib/libc/string/wcscpy.c \
-    upstream-freebsd/lib/libc/string/wcscat.c \
-    upstream-openbsd/lib/libc/string/strlcat.c \
-    upstream-openbsd/lib/libc/string/strlcpy.c \
-    upstream-openbsd/lib/libc/string/strncat.c
-endif
-
-ifeq ($(ARCH_X86_HAVE_SSE4),true)
- libc_bionic_src_files_x86 += \
-    arch-x86/silvermont/string/sse4-memcmp-slm.S \
-    arch-x86/silvermont/string/sse4-wmemcmp-slm.S
-else
-libc_bionic_src_files_x86 += \
-    arch-x86/generic/string/memcmp.S \
-    upstream-freebsd/lib/libc/string/wmemcmp.c
-endif

libc/arch-x86/silvermont/silvermont.mk

@@ -1,32 +0,0 @@
-libc_bionic_src_files_x86 += \
-    arch-x86/silvermont/string/sse2-bcopy-slm.S \
-    arch-x86/silvermont/string/sse2-bzero-slm.S \
-    arch-x86/silvermont/string/sse2-memcpy-slm.S \
-    arch-x86/silvermont/string/sse2-memmove-slm.S \
-    arch-x86/silvermont/string/sse2-memset-slm.S \
-    arch-x86/silvermont/string/sse2-stpcpy-slm.S \
-    arch-x86/silvermont/string/sse2-stpncpy-slm.S \
-    arch-x86/silvermont/string/sse2-strcpy-slm.S \
-    arch-x86/silvermont/string/sse2-strlen-slm.S \
-    arch-x86/silvermont/string/sse2-strncpy-slm.S \
-    arch-x86/silvermont/string/sse4-memcmp-slm.S \
-    arch-x86/silvermont/string/sse4-wmemcmp-slm.S
-
-libc_bionic_src_files_x86 += \
-    arch-x86/atom/string/sse2-memchr-atom.S \
-    arch-x86/atom/string/sse2-memrchr-atom.S \
-    arch-x86/atom/string/sse2-strchr-atom.S \
-    arch-x86/atom/string/sse2-strrchr-atom.S \
-    arch-x86/atom/string/sse2-strnlen-atom.S \
-    arch-x86/atom/string/sse2-wcschr-atom.S \
-    arch-x86/atom/string/sse2-wcsrchr-atom.S \
-    arch-x86/atom/string/sse2-wcslen-atom.S \
-    arch-x86/atom/string/sse2-wcscmp-atom.S \
-    arch-x86/atom/string/ssse3-strncat-atom.S \
-    arch-x86/atom/string/ssse3-strlcat-atom.S \
-    arch-x86/atom/string/ssse3-strlcpy-atom.S \
-    arch-x86/atom/string/ssse3-strcmp-atom.S \
-    arch-x86/atom/string/ssse3-strncmp-atom.S \
-    arch-x86/atom/string/ssse3-strcat-atom.S \
-    arch-x86/atom/string/ssse3-wcscat-atom.S \
-    arch-x86/atom/string/ssse3-wcscpy-atom.S

libc/arch-x86/syscalls/__clock_gettime.S

@@ -2,7 +2,7 @@
 
 #include <private/bionic_asm.h>
 
-ENTRY(clock_gettime)
+ENTRY(__clock_gettime)
     pushl   %ebx
     .cfi_def_cfa_offset 8
     .cfi_rel_offset ebx, 0
@@ -23,4 +23,4 @@ ENTRY(clock_gettime)
     popl    %ecx
     popl    %ebx
     ret
-END(clock_gettime)
+END(__clock_gettime)

libc/arch-x86/syscalls/__gettimeofday.S

@@ -2,7 +2,7 @@
 
 #include <private/bionic_asm.h>
 
-ENTRY(gettimeofday)
+ENTRY(__gettimeofday)
     pushl   %ebx
     .cfi_def_cfa_offset 8
     .cfi_rel_offset ebx, 0
@@ -23,4 +23,4 @@ ENTRY(gettimeofday)
     popl    %ecx
     popl    %ebx
     ret
-END(gettimeofday)
+END(__gettimeofday)

libc/arch-x86/syscalls/__preadv64.S

@@ -0,0 +1,41 @@
+/* Generated by gensyscalls.py. Do not edit. */
+
+#include <private/bionic_asm.h>
+
+ENTRY(__preadv64)
+    pushl   %ebx
+    .cfi_def_cfa_offset 8
+    .cfi_rel_offset ebx, 0
+    pushl   %ecx
+    .cfi_adjust_cfa_offset 4
+    .cfi_rel_offset ecx, 0
+    pushl   %edx
+    .cfi_adjust_cfa_offset 4
+    .cfi_rel_offset edx, 0
+    pushl   %esi
+    .cfi_adjust_cfa_offset 4
+    .cfi_rel_offset esi, 0
+    pushl   %edi
+    .cfi_adjust_cfa_offset 4
+    .cfi_rel_offset edi, 0
+    mov     24(%esp), %ebx
+    mov     28(%esp), %ecx
+    mov     32(%esp), %edx
+    mov     36(%esp), %esi
+    mov     40(%esp), %edi
+    movl    $__NR_preadv, %eax
+    int     $0x80
+    cmpl    $-MAX_ERRNO, %eax
+    jb      1f
+    negl    %eax
+    pushl   %eax
+    call    __set_errno_internal
+    addl    $4, %esp
+1:
+    popl    %edi
+    popl    %esi
+    popl    %edx
+    popl    %ecx
+    popl    %ebx
+    ret
+END(__preadv64)

libc/arch-x86/syscalls/__pwritev64.S

@@ -0,0 +1,41 @@
+/* Generated by gensyscalls.py. Do not edit. */
+
+#include <private/bionic_asm.h>
+
+ENTRY(__pwritev64)
+    pushl   %ebx
+    .cfi_def_cfa_offset 8
+    .cfi_rel_offset ebx, 0
+    pushl   %ecx
+    .cfi_adjust_cfa_offset 4
+    .cfi_rel_offset ecx, 0
+    pushl   %edx
+    .cfi_adjust_cfa_offset 4
+    .cfi_rel_offset edx, 0
+    pushl   %esi
+    .cfi_adjust_cfa_offset 4
+    .cfi_rel_offset esi, 0
+    pushl   %edi
+    .cfi_adjust_cfa_offset 4
+    .cfi_rel_offset edi, 0
+    mov     24(%esp), %ebx
+    mov     28(%esp), %ecx
+    mov     32(%esp), %edx
+    mov     36(%esp), %esi
+    mov     40(%esp), %edi
+    movl    $__NR_pwritev, %eax
+    int     $0x80
+    cmpl    $-MAX_ERRNO, %eax
+    jb      1f
+    negl    %eax
+    pushl   %eax
+    call    __set_errno_internal
+    addl    $4, %esp
+1:
+    popl    %edi
+    popl    %esi
+    popl    %edx
+    popl    %ecx
+    popl    %ebx
+    ret
+END(__pwritev64)

libc/arch-x86/x86.mk

@@ -1,17 +1,103 @@
 # 32-bit x86.
 
 #
-# Default implementations of functions that are commonly optimized.
+# Generic x86 optimizations, may be overriden by CPU variants.
 #
 
-libc_common_src_files_x86 += \
-    bionic/__memcpy_chk.cpp \
-    bionic/__memset_chk.cpp \
-    bionic/__strcpy_chk.cpp \
-    bionic/__strcat_chk.cpp \
+libc_bionic_src_files_x86 += \
+    arch-x86/atom/string/sse2-memchr-atom.S \
+    arch-x86/atom/string/sse2-memrchr-atom.S \
+    arch-x86/atom/string/sse2-strchr-atom.S \
+    arch-x86/atom/string/sse2-strnlen-atom.S \
+    arch-x86/atom/string/sse2-strrchr-atom.S \
+    arch-x86/atom/string/sse2-wcschr-atom.S \
+    arch-x86/atom/string/sse2-wcsrchr-atom.S \
+    arch-x86/atom/string/sse2-wcslen-atom.S \
+    arch-x86/atom/string/sse2-wcscmp-atom.S \
+    arch-x86/silvermont/string/sse2-bcopy-slm.S \
+    arch-x86/silvermont/string/sse2-bzero-slm.S \
+    arch-x86/silvermont/string/sse2-memcpy-slm.S \
+    arch-x86/silvermont/string/sse2-memmove-slm.S \
+    arch-x86/silvermont/string/sse2-memset-slm.S \
+    arch-x86/silvermont/string/sse2-stpcpy-slm.S \
+    arch-x86/silvermont/string/sse2-stpncpy-slm.S \
+    arch-x86/silvermont/string/sse2-strcpy-slm.S \
+    arch-x86/silvermont/string/sse2-strlen-slm.S \
+    arch-x86/silvermont/string/sse2-strncpy-slm.S
 
-libc_freebsd_src_files_x86 += \
-    upstream-freebsd/lib/libc/string/wmemmove.c \
+libc_bionic_src_files_x86 += \
+    arch-x86/generic/string/memcmp.S \
+    arch-x86/generic/string/strcmp.S \
+    arch-x86/generic/string/strncmp.S \
+    arch-x86/generic/string/strcat.S
+
+ifeq ($(ARCH_X86_HAVE_SSSE3),true)
+libc_bionic_src_files_x86 += \
+    arch-x86/atom/string/ssse3-strncat-atom.S \
+    arch-x86/atom/string/ssse3-strlcat-atom.S \
+    arch-x86/atom/string/ssse3-strlcpy-atom.S \
+    arch-x86/atom/string/ssse3-strcmp-atom.S \
+    arch-x86/atom/string/ssse3-strncmp-atom.S \
+    arch-x86/atom/string/ssse3-strcat-atom.S \
+    arch-x86/atom/string/ssse3-wcscat-atom.S \
+    arch-x86/atom/string/ssse3-wcscpy-atom.S
+libc_bionic_src_files_exclude_x86 += \
+    arch-x86/generic/string/strcmp.S \
+    arch-x86/generic/string/strncmp.S \
+    arch-x86/generic/string/strcat.S
+endif
+
+ifeq ($(ARCH_X86_HAVE_SSE4),true)
+libc_bionic_src_files_x86 += \
+    arch-x86/silvermont/string/sse4-memcmp-slm.S \
+    arch-x86/silvermont/string/sse4-wmemcmp-slm.S
+libc_bionic_src_files_exclude_x86 += \
+    arch-x86/generic/string/memcmp.S
+endif
+
+#
+# Remove default implementations that we have optimized versions of.
+#
+
+libc_freebsd_src_files_exclude_x86 += \
+    upstream-freebsd/lib/libc/string/wcschr.c \
+    upstream-freebsd/lib/libc/string/wcscmp.c \
+    upstream-freebsd/lib/libc/string/wcslen.c \
+    upstream-freebsd/lib/libc/string/wcsrchr.c \
+
+ifeq ($(ARCH_X86_HAVE_SSSE3),true)
+libc_freebsd_src_files_exclude_x86 += \
+    upstream-freebsd/lib/libc/string/wcscat.c \
+    upstream-freebsd/lib/libc/string/wcscpy.c
+endif
+
+ifeq ($(ARCH_X86_HAVE_SSE4),true)
+libc_freebsd_src_files_exclude_x86 += \
+    upstream-freebsd/lib/libc/string/wmemcmp.c
+endif
+
+libc_openbsd_src_files_exclude_x86 += \
+    upstream-openbsd/lib/libc/string/memchr.c \
+    upstream-openbsd/lib/libc/string/memmove.c \
+    upstream-openbsd/lib/libc/string/memrchr.c \
+    upstream-openbsd/lib/libc/string/stpcpy.c \
+    upstream-openbsd/lib/libc/string/stpncpy.c \
+    upstream-openbsd/lib/libc/string/strcat.c \
+    upstream-openbsd/lib/libc/string/strcpy.c \
+    upstream-openbsd/lib/libc/string/strncmp.c \
+    upstream-openbsd/lib/libc/string/strncpy.c \
+
+ifeq ($(ARCH_X86_HAVE_SSSE3),true)
+libc_openbsd_src_files_exclude_x86 += \
+    upstream-openbsd/lib/libc/string/strlcat.c \
+    upstream-openbsd/lib/libc/string/strlcpy.c \
+    upstream-openbsd/lib/libc/string/strncat.c
+endif
+
+libc_bionic_src_files_exclude_x86 += \
+    bionic/strchr.cpp \
+    bionic/strnlen.c \
+    bionic/strrchr.cpp \
 
 #
 # Inherently architecture-specific functions.
@@ -30,12 +116,14 @@ libc_bionic_src_files_x86 += \
 ## ARCH variant specific source files
 arch_variant_mk := $(LOCAL_PATH)/arch-x86/$(TARGET_ARCH_VARIANT)/$(TARGET_ARCH_VARIANT).mk
 ifeq ($(wildcard $(arch_variant_mk)),)
-    arch_variant_mk := $(LOCAL_PATH)/arch-x86/generic/generic.mk
+    arch_variant_mk :=
 endif
+ifneq ($(arch_variant_mk),)
 include $(arch_variant_mk)
 libc_common_additional_dependencies += $(arch_variant_mk)
 
 arch_variant_mk :=
+endif
 
 libc_crt_target_cflags_x86 := \
     -m32 \

libc/arch-x86_64/bionic/setjmp.S

@@ -50,6 +50,25 @@
 #define _JB_SIGMASK 9
 #define _JB_SIGMASK_RT 10 // sigprocmask will write here too.
 
+#define MANGLE_REGISTERS 1
+.macro m_mangle_registers reg
+#if MANGLE_REGISTERS
+  xorq \reg,%rbx
+  xorq \reg,%rbp
+  xorq \reg,%r12
+  xorq \reg,%r13
+  xorq \reg,%r14
+  xorq \reg,%r15
+  xorq \reg,%rsp
+  xorq \reg,%r11
+#endif
+.endm
+
+.macro m_unmangle_registers reg
+  m_mangle_registers \reg
+.endm
+
+
 ENTRY(setjmp)
   movl $1,%esi
   jmp PIC_PLT(sigsetjmp)
@@ -62,11 +81,17 @@ END(_setjmp)
 
 // int sigsetjmp(sigjmp_buf env, int save_signal_mask);
 ENTRY(sigsetjmp)
-  // Record whether or not we're saving the signal mask.
-  movl %esi,(_JB_SIGFLAG * 8)(%rdi)
+  pushq %rdi
+  movq %rsi,%rdi
+  call PIC_PLT(__bionic_setjmp_cookie_get)
+  popq %rdi
+
+  // Record setjmp cookie and whether or not we're saving the signal mask.
+  movq %rax,(_JB_SIGFLAG * 8)(%rdi)
+  pushq %rax
 
   // Do we need to save the signal mask?
-  testl %esi,%esi
+  testq $1,%rax
   jz 2f
 
   // Save current signal mask.
@@ -79,7 +104,10 @@ ENTRY(sigsetjmp)
 
 2:
   // Save the callee-save registers.
+  popq %rax
+  andq $-2,%rax
   movq (%rsp),%r11
+  m_mangle_registers %rax
   movq %rbx,(_JB_RBX * 8)(%rdi)
   movq %rbp,(_JB_RBP * 8)(%rdi)
   movq %r12,(_JB_R12 * 8)(%rdi)
@@ -88,6 +116,7 @@ ENTRY(sigsetjmp)
   movq %r15,(_JB_R15 * 8)(%rdi)
   movq %rsp,(_JB_RSP * 8)(%rdi)
   movq %r11,(_JB_PC  * 8)(%rdi)
+  m_unmangle_registers %rax
 
   xorl %eax,%eax
   ret
@@ -99,7 +128,9 @@ ENTRY(siglongjmp)
   pushq %rsi // Push 'value'.
 
   // Do we need to restore the signal mask?
-  cmpl $0,(_JB_SIGFLAG * 8)(%rdi)
+  movq (_JB_SIGFLAG * 8)(%rdi), %rdi
+  pushq %rdi // Push cookie
+  testq $1, %rdi
   jz 2f
 
   // Restore the signal mask.
@@ -109,6 +140,10 @@ ENTRY(siglongjmp)
   call PIC_PLT(sigprocmask)
 
 2:
+  // Fetch the setjmp cookie and clear the signal flag bit.
+  popq %rcx
+  andq $-2, %rcx
+
   popq %rax // Pop 'value'.
 
   // Restore the callee-save registers.
@@ -120,7 +155,17 @@ ENTRY(siglongjmp)
   movq (_JB_RSP * 8)(%r12),%rsp
   movq (_JB_PC  * 8)(%r12),%r11
   movq (_JB_R12 * 8)(%r12),%r12
+  m_unmangle_registers %rcx
 
+  // Check the cookie.
+  pushq %rax
+  pushq %r11
+  movq %rcx, %rdi
+  call PIC_PLT(__bionic_setjmp_cookie_check)
+  popq %r11
+  popq %rax
+
+  // Return 1 if value is 0.
   testl %eax,%eax
   jnz 1f
   incl %eax

libc/arch-x86_64/bionic/vfork.S

@@ -32,6 +32,12 @@
 
 ENTRY(vfork)
   popq    %rdi  // Grab the return address.
+
+  // __get_tls()[TLS_SLOT_THREAD_ID]->cached_pid_ = 0
+  mov    %fs:0, %rax
+  mov    8(%rax), %rax
+  movl   $0, 20(%rax)
+
   movl    $__NR_vfork, %eax
   syscall
   pushq   %rdi  // Restore the return address.

libc/arch-x86_64/string/sse2-memmove-slm.S

@@ -91,9 +91,6 @@ name:		\
 	.section .text.sse2,"ax",@progbits
 ENTRY (MEMMOVE)
 	ENTRANCE
-#ifdef USE_AS_BCOPY
-	xchg	%rsi, %rdi
-#endif
 	mov	%rdi, %rax
 
 /* Check whether we should copy backward or forward.  */

libc/arch-x86_64/syscalls/preadv.S

@@ -0,0 +1,18 @@
+/* Generated by gensyscalls.py. Do not edit. */
+
+#include <private/bionic_asm.h>
+
+ENTRY(preadv)
+    movq    %rcx, %r10
+    movl    $__NR_preadv, %eax
+    syscall
+    cmpq    $-MAX_ERRNO, %rax
+    jb      1f
+    negl    %eax
+    movl    %eax, %edi
+    call    __set_errno_internal
+1:
+    ret
+END(preadv)
+
+ALIAS_SYMBOL(preadv64, preadv)

libc/arch-x86_64/syscalls/pwritev.S

@@ -0,0 +1,18 @@
+/* Generated by gensyscalls.py. Do not edit. */
+
+#include <private/bionic_asm.h>
+
+ENTRY(pwritev)
+    movq    %rcx, %r10
+    movl    $__NR_pwritev, %eax
+    syscall
+    cmpq    $-MAX_ERRNO, %rax
+    jb      1f
+    negl    %eax
+    movl    %eax, %edi
+    call    __set_errno_internal
+1:
+    ret
+END(pwritev)
+
+ALIAS_SYMBOL(pwritev64, pwritev)

libc/arch-x86_64/x86_64.mk

@@ -1,31 +1,20 @@
 # 64-bit x86.
 
 #
-# Default implementations of functions that are commonly optimized.
+# Remove default implementations that we have optimized versions of.
 #
 
-libc_bionic_src_files_x86_64 += \
-    bionic/__memcpy_chk.cpp \
-    bionic/__memset_chk.cpp \
-    bionic/__strcpy_chk.cpp \
-    bionic/__strcat_chk.cpp \
-    bionic/strchr.cpp \
-    bionic/strnlen.c \
-    bionic/strrchr.cpp \
-
-libc_freebsd_src_files_x86_64 += \
-    upstream-freebsd/lib/libc/string/wcscat.c \
-    upstream-freebsd/lib/libc/string/wcschr.c \
-    upstream-freebsd/lib/libc/string/wcscmp.c \
-    upstream-freebsd/lib/libc/string/wcscpy.c \
-    upstream-freebsd/lib/libc/string/wcslen.c \
-    upstream-freebsd/lib/libc/string/wcsrchr.c \
-    upstream-freebsd/lib/libc/string/wmemcmp.c \
-    upstream-freebsd/lib/libc/string/wmemmove.c \
-
-libc_openbsd_src_files_x86_64 += \
-    upstream-openbsd/lib/libc/string/memchr.c \
-    upstream-openbsd/lib/libc/string/memrchr.c \
+libc_openbsd_src_files_exclude_x86_64 += \
+    upstream-openbsd/lib/libc/string/memmove.c \
+    upstream-openbsd/lib/libc/string/stpcpy.c \
+    upstream-openbsd/lib/libc/string/stpncpy.c \
+    upstream-openbsd/lib/libc/string/strcat.c \
+    upstream-openbsd/lib/libc/string/strcpy.c \
+    upstream-openbsd/lib/libc/string/strlcat.c \
+    upstream-openbsd/lib/libc/string/strlcpy.c \
+    upstream-openbsd/lib/libc/string/strncat.c \
+    upstream-openbsd/lib/libc/string/strncmp.c \
+    upstream-openbsd/lib/libc/string/strncpy.c \
 
 #
 # Inherently architecture-specific code.

libc/bionic/__fread_chk.cpp

@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#undef _FORTIFY_SOURCE
+#include <stdio.h>
+#include <sys/cdefs.h>
+#include "private/libc_logging.h"
+
+extern "C" size_t __fread_chk(void * __restrict buf, size_t size, size_t count,
+                              FILE * __restrict stream, size_t buf_size) {
+  size_t total;
+  if (__predict_false(__size_mul_overflow(size, count, &total))) {
+    // overflow: trigger the error path in fread
+    return fread(buf, size, count, stream);
+  }
+
+  if (__predict_false(total > buf_size)) {
+    __fortify_chk_fail("fread: prevented write past end of buffer", 0);
+  }
+
+  return fread(buf, size, count, stream);
+}

libc/bionic/__fwrite_chk.cpp

@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#undef _FORTIFY_SOURCE
+#include <stdio.h>
+#include <sys/cdefs.h>
+#include "private/libc_logging.h"
+
+extern "C" size_t __fwrite_chk(const void * __restrict buf, size_t size, size_t count,
+                               FILE * __restrict stream, size_t buf_size) {
+  size_t total;
+  if (__predict_false(__size_mul_overflow(size, count, &total))) {
+    // overflow: trigger the error path in fwrite
+    return fwrite(buf, size, count, stream);
+  }
+
+  if (__predict_false(total > buf_size)) {
+    __fortify_chk_fail("fwrite: prevented read past end of buffer", 0);
+  }
+
+  return fwrite(buf, size, count, stream);
+}

libc/bionic/__getcwd_chk.cpp

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#undef _FORTIFY_SOURCE
+
+#include <unistd.h>
+#include "private/libc_logging.h"
+
+extern char* __getcwd_chk(char* buf, size_t len, size_t buflen) {
+  if (__predict_false(len > buflen)) {
+    __fortify_chk_fail("getcwd: prevented write past end of buffer", 0);
+  }
+
+  return getcwd(buf, len);
+}

libc/bionic/__pwrite64_chk.cpp

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#undef _FORTIFY_SOURCE
+#include <unistd.h>
+#include "private/libc_logging.h"
+
+extern "C" ssize_t __pwrite64_chk(int fd, const void* buf, size_t count, off64_t offset,
+                                  size_t buf_size) {
+  if (__predict_false(count > buf_size)) {
+    __fortify_chk_fail("pwrite64: prevented read past end of buffer", 0);
+  }
+
+  if (__predict_false(count > SSIZE_MAX)) {
+    __fortify_chk_fail("pwrite64: count > SSIZE_MAX", 0);
+  }
+
+  return pwrite64(fd, buf, count, offset);
+}

libc/bionic/__pwrite_chk.cpp

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#undef _FORTIFY_SOURCE
+#include <unistd.h>
+#include "private/libc_logging.h"
+
+extern "C" ssize_t __pwrite_chk(int fd, const void* buf, size_t count, off_t offset,
+                                size_t buf_size) {
+  if (__predict_false(count > buf_size)) {
+    __fortify_chk_fail("pwrite: prevented read past end of buffer", 0);
+  }
+
+  if (__predict_false(count > SSIZE_MAX)) {
+    __fortify_chk_fail("pwrite: count > SSIZE_MAX", 0);
+  }
+
+  return pwrite(fd, buf, count, offset);
+}

libc/bionic/__set_errno.cpp

@@ -36,20 +36,13 @@
 // system these are the same size, but on a 64-bit system they're not.
 // 'long' gives us 32-bit on 32-bit systems, 64-bit on 64-bit systems.
 
-// __set_errno was mistakenly exposed in <errno.h> in the 32-bit NDK.
-// We need the extra level of indirection so that the .hidden directives
-// in the system call stubs don't cause __set_errno to be hidden, breaking
-// old NDK apps.
+// Since __set_errno was mistakenly exposed in <errno.h> in the 32-bit
+// NDK, use a differently named internal function for the system call
+// stubs. This avoids having the stubs .hidden directives accidentally
+// hide __set_errno for old NDK apps.
 
 // This one is for internal use only and used by both LP32 and LP64 assembler.
 extern "C" __LIBC_HIDDEN__ long __set_errno_internal(int n) {
   errno = n;
   return -1;
 }
-
-// This one exists for the LP32 NDK and is not present at all in LP64.
-#if !defined(__LP64__)
-extern "C" long __set_errno(int n) {
-  return __set_errno_internal(n);
-}
-#endif

libc/bionic/__write_chk.cpp

@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+ * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#undef _FORTIFY_SOURCE
+#include <unistd.h>
+#include "private/libc_logging.h"
+
+extern "C" ssize_t __write_chk(int fd, const void* buf, size_t count, size_t buf_size) {
+  if (__predict_false(count > buf_size)) {
+    __fortify_chk_fail("write: prevented read past end of buffer", 0);
+  }
+
+  if (__predict_false(count > SSIZE_MAX)) {
+    __fortify_chk_fail("write: count > SSIZE_MAX", 0);
+  }
+
+  return write(fd, buf, count);
+}

libc/bionic/arpa_inet.cpp

@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <stdlib.h>
+
+#include "private/ErrnoRestorer.h"
+
+// The difference between inet_network(3) and inet_addr(3) is that
+// inet_network uses host order and inet_addr network order.
+in_addr_t inet_network(const char* cp) {
+  in_addr_t network_order = inet_addr(cp);
+  return ntohl(network_order);
+}
+
+in_addr_t inet_addr(const char* cp) {
+  in_addr addr;
+  return inet_aton(cp, &addr) ? addr.s_addr : INADDR_NONE;
+}
+
+int inet_aton(const char* cp, in_addr* addr) {
+  ErrnoRestorer errno_restorer;
+
+  unsigned long parts[4];
+  size_t i;
+  for (i = 0; i < 4; ++i) {
+    char* end;
+    errno = 0;
+    parts[i] = strtoul(cp, &end, 0);
+    if (errno != 0 || end == cp || (*end != '.' && *end != '\0')) return 0;
+    if (*end == '\0') break;
+    cp = end + 1;
+  }
+
+  uint32_t result = 0;
+  if (i == 0) {
+    // a (a 32-bit).
+    if (parts[0] > 0xffffffff) return 0;
+    result = parts[0];
+  } else if (i == 1) {
+    // a.b (b 24-bit).
+    if (parts[0] > 0xff || parts[1] > 0xffffff) return 0;
+    result = (parts[0] << 24) | parts[1];
+  } else if (i == 2) {
+    // a.b.c (c 16-bit).
+    if (parts[0] > 0xff || parts[1] > 0xff || parts[2] > 0xffff) return 0;
+    result = (parts[0] << 24) | (parts[1] << 16) | parts[2];
+  } else if (i == 3) {
+    // a.b.c.d (d 8-bit).
+    if (parts[0] > 0xff || parts[1] > 0xff || parts[2] > 0xff || parts[3] > 0xff) return 0;
+    result = (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3];
+  } else {
+    return 0;
+  }
+
+  if (addr != nullptr) addr->s_addr = htonl(result);
+  return 1;
+}

libc/bionic/bionic_systrace.cpp

@@ -21,6 +21,7 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include "private/bionic_lock.h"
 #include "private/bionic_systrace.h"
 #include "private/libc_logging.h"
 
@@ -29,12 +30,17 @@
 
 #define WRITE_OFFSET   32
 
-static const prop_info* g_pinfo = NULL;
+constexpr char SYSTRACE_PROPERTY_NAME[] = "debug.atrace.tags.enableflags";
+
+static Lock g_lock;
+static const prop_info* g_pinfo;
 static uint32_t g_serial = -1;
-static uint64_t g_tags = 0;
+static uint64_t g_tags;
 static int g_trace_marker_fd = -1;
 
 static bool should_trace() {
+  bool result = false;
+  g_lock.lock();
   // If g_pinfo is null, this means that systrace hasn't been run and it's safe to
   // assume that no trace writing will need to take place.  However, to avoid running
   // this costly find check each time, we set it to a non-tracing value so that next
@@ -42,32 +48,39 @@ static bool should_trace() {
   // this function also deals with the bootup case, during which the call to property
   // set will fail if the property server hasn't yet started.
   if (g_pinfo == NULL) {
-    g_pinfo = __system_property_find("debug.atrace.tags.enableflags");
+    g_pinfo = __system_property_find(SYSTRACE_PROPERTY_NAME);
     if (g_pinfo == NULL) {
-      __system_property_set("debug.atrace.tags.enableflags", "0");
-      g_pinfo = __system_property_find("debug.atrace.tags.enableflags");
-      if (g_pinfo == NULL) {
-        return false;
+      __system_property_set(SYSTRACE_PROPERTY_NAME, "0");
+      g_pinfo = __system_property_find(SYSTRACE_PROPERTY_NAME);
     }
   }
-  }
-
+  if (g_pinfo != NULL) {
     // Find out which tags have been enabled on the command line and set
     // the value of tags accordingly.  If the value of the property changes,
     // the serial will also change, so the costly system_property_read function
     // can be avoided by calling the much cheaper system_property_serial
     // first.  The values within pinfo may change, but its location is guaranteed
     // not to move.
-  const uint32_t cur_serial = __system_property_serial(g_pinfo);
+    uint32_t cur_serial = __system_property_serial(g_pinfo);
     if (cur_serial != g_serial) {
       g_serial = cur_serial;
       char value[PROP_VALUE_MAX];
       __system_property_read(g_pinfo, 0, value);
       g_tags = strtoull(value, NULL, 0);
     }
+    result = ((g_tags & ATRACE_TAG_BIONIC) != 0);
+  }
+  g_lock.unlock();
+  return result;
+}
 
-  // Finally, verify that this tag value enables bionic tracing.
-  return ((g_tags & ATRACE_TAG_BIONIC) != 0);
+static int get_trace_marker_fd() {
+  g_lock.lock();
+  if (g_trace_marker_fd == -1) {
+    g_trace_marker_fd = open("/sys/kernel/debug/tracing/trace_marker", O_CLOEXEC | O_WRONLY);
+  }
+  g_lock.unlock();
+  return g_trace_marker_fd;
 }
 
 ScopedTrace::ScopedTrace(const char* message) {
@@ -75,11 +88,9 @@ ScopedTrace::ScopedTrace(const char* message) {
     return;
   }
 
-  if (g_trace_marker_fd == -1) {
-    g_trace_marker_fd = open("/sys/kernel/debug/tracing/trace_marker", O_CLOEXEC | O_WRONLY);
-    if (g_trace_marker_fd == -1) {
-      __libc_fatal("Could not open kernel trace file: %s\n", strerror(errno));
-    }
+  int trace_marker_fd = get_trace_marker_fd();
+  if (trace_marker_fd == -1) {
+    return;
   }
 
   // If bionic tracing has been enabled, then write the message to the
@@ -87,12 +98,10 @@ ScopedTrace::ScopedTrace(const char* message) {
   int length = strlen(message);
   char buf[length + WRITE_OFFSET];
   size_t len = snprintf(buf, length + WRITE_OFFSET, "B|%d|%s", getpid(), message);
-  ssize_t wbytes = TEMP_FAILURE_RETRY(write(g_trace_marker_fd, buf, len));
 
-  // Error while writing
-  if (static_cast<size_t>(wbytes) != len) {
-    __libc_fatal("Could not write to kernel trace file: %s\n", strerror(errno));
-  }
+  // Tracing may stop just after checking property and before writing the message.
+  // So the write is acceptable to fail. See b/20666100.
+  TEMP_FAILURE_RETRY(write(trace_marker_fd, buf, len));
 }
 
 ScopedTrace::~ScopedTrace() {
@@ -100,10 +109,10 @@ ScopedTrace::~ScopedTrace() {
     return;
   }
 
-  ssize_t wbytes = TEMP_FAILURE_RETRY(write(g_trace_marker_fd, "E", 1));
-
-  // Error while writing
-  if (static_cast<size_t>(wbytes) != 1) {
-    __libc_fatal("Could not write to kernel trace file: %s\n", strerror(errno));
+  int trace_marker_fd = get_trace_marker_fd();
+  if (trace_marker_fd == -1) {
+    return;
   }
+
+  TEMP_FAILURE_RETRY(write(trace_marker_fd, "E", 1));
 }

libc/bionic/debug_backtrace.cpp

@@ -26,7 +26,7 @@
  * SUCH DAMAGE.
  */
 
-#include "debug_stacktrace.h"
+#include "debug_backtrace.h"
 
 #include <dlfcn.h>
 #include <inttypes.h>
@@ -85,23 +85,37 @@ static _Unwind_Reason_Code trace_function(__unwind_context* context, void* arg)
     return _URC_NO_REASON;
   }
 
-#if defined(__arm__)
-  /*
-   * The instruction pointer is pointing at the instruction after the bl(x), and
-   * the _Unwind_Backtrace routine already masks the Thumb mode indicator (LSB
-   * in PC). So we need to do a quick check here to find out if the previous
-   * instruction is a Thumb-mode BLX(2). If so subtract 2 otherwise 4 from PC.
-   */
+  // The instruction pointer is pointing at the instruction after the return
+  // call on all architectures.
+  // Modify the pc to point at the real function.
   if (ip != 0) {
-    short* ptr = reinterpret_cast<short*>(ip);
-    // Thumb BLX(2)
-    if ((*(ptr-1) & 0xff80) == 0x4780) {
-      ip -= 2;
-    } else {
+#if defined(__arm__)
+    // If the ip is suspiciously low, do nothing to avoid a segfault trying
+    // to access this memory.
+    if (ip >= 4096) {
+      // Check bits [15:11] of the first halfword assuming the instruction
+      // is 32 bits long. If the bits are any of these values, then our
+      // assumption was correct:
+      //  b11101
+      //  b11110
+      //  b11111
+      // Otherwise, this is a 16 bit instruction.
+      uint16_t value = (*reinterpret_cast<uint16_t*>(ip - 2)) >> 11;
+      if (value == 0x1f || value == 0x1e || value == 0x1d) {
         ip -= 4;
+      } else {
+        ip -= 2;
       }
     }
+#elif defined(__aarch64__)
+    // All instructions are 4 bytes long, skip back one instruction.
+    ip -= 4;
+#elif defined(__i386__) || defined(__x86_64__)
+    // It's difficult to decode exactly where the previous instruction is,
+    // so subtract 1 to estimate where the instruction lives.
+    ip--;
 #endif
+  }
 
   state->frames[state->frame_count++] = ip;
   return (state->frame_count >= state->max_depth) ? _URC_END_OF_STACK : _URC_NO_REASON;

libc/bionic/debug_backtrace.h

@@ -26,8 +26,8 @@
  * SUCH DAMAGE.
  */
 
-#ifndef DEBUG_STACKTRACE_H
-#define DEBUG_STACKTRACE_H
+#ifndef DEBUG_BACKTRACE_H
+#define DEBUG_BACKTRACE_H
 
 #include <stdint.h>
 #include <sys/cdefs.h>
@@ -37,4 +37,4 @@ __LIBC_HIDDEN__ void backtrace_shutdown();
 __LIBC_HIDDEN__ int get_backtrace(uintptr_t* stack_frames, size_t max_depth);
 __LIBC_HIDDEN__ void log_backtrace(uintptr_t* stack_frames, size_t frame_count);
 
-#endif /* DEBUG_STACKTRACE_H */
+#endif /* DEBUG_BACKTRACE_H */

libc/bionic/debug_mapinfo.cpp

@@ -27,6 +27,7 @@
  */
 
 #include <ctype.h>
+#include <elf.h>
 #include <inttypes.h>
 #include <stdio.h>
 #include <string.h>
@@ -35,20 +36,25 @@
 #include "debug_mapinfo.h"
 #include "malloc_debug_disable.h"
 
+#if defined(__LP64__)
+#define Elf_W(x) Elf64_##x
+#else
+#define Elf_W(x) Elf32_##x
+#endif
+
 // Format of /proc/<PID>/maps:
 //   6f000000-6f01e000 rwxp 00000000 00:0c 16389419   /system/lib/libcomposer.so
 static mapinfo_t* parse_maps_line(char* line) {
   uintptr_t start;
   uintptr_t end;
+  uintptr_t offset;
+  char permissions[4];
   int name_pos;
-  if (sscanf(line, "%" PRIxPTR "-%" PRIxPTR " %*4s %*x %*x:%*x %*d%n", &start,
-             &end, &name_pos) < 2) {
+  if (sscanf(line, "%" PRIxPTR "-%" PRIxPTR " %4s %" PRIxPTR " %*x:%*x %*d %n", &start,
+             &end, permissions, &offset, &name_pos) < 2) {
     return NULL;
   }
 
-  while (isspace(line[name_pos])) {
-    name_pos += 1;
-  }
   const char* name = line + name_pos;
   size_t name_len = strlen(name);
   if (name_len && name[name_len - 1] == '\n') {
@@ -59,6 +65,14 @@ static mapinfo_t* parse_maps_line(char* line) {
   if (mi) {
     mi->start = start;
     mi->end = end;
+    mi->offset = offset;
+    if (permissions[0] != 'r') {
+      // Any unreadable map will just get a zero load base.
+      mi->load_base = 0;
+      mi->load_base_read = true;
+    } else {
+      mi->load_base_read = false;
+    }
     memcpy(mi->name, name, name_len);
     mi->name[name_len] = '\0';
   }
@@ -95,11 +109,58 @@ __LIBC_HIDDEN__ void mapinfo_destroy(mapinfo_t* mi) {
   }
 }
 
+template<typename T>
+static inline bool get_val(mapinfo_t* mi, uintptr_t addr, T* store) {
+  if (addr < mi->start || addr + sizeof(T) > mi->end) {
+    return false;
+  }
+  // Make sure the address is aligned properly.
+  if (addr & (sizeof(T)-1)) {
+    return false;
+  }
+  *store = *reinterpret_cast<T*>(addr);
+  return true;
+}
+
+__LIBC_HIDDEN__ void mapinfo_read_loadbase(mapinfo_t* mi) {
+  mi->load_base = 0;
+  mi->load_base_read = true;
+  uintptr_t addr = mi->start;
+  Elf_W(Ehdr) ehdr;
+  if (!get_val<Elf_W(Half)>(mi, addr + offsetof(Elf_W(Ehdr), e_phnum), &ehdr.e_phnum)) {
+    return;
+  }
+  if (!get_val<Elf_W(Off)>(mi, addr + offsetof(Elf_W(Ehdr), e_phoff), &ehdr.e_phoff)) {
+    return;
+  }
+  addr += ehdr.e_phoff;
+  for (size_t i = 0; i < ehdr.e_phnum; i++) {
+    Elf_W(Phdr) phdr;
+    if (!get_val<Elf_W(Word)>(mi, addr + offsetof(Elf_W(Phdr), p_type), &phdr.p_type)) {
+      return;
+    }
+    if (!get_val<Elf_W(Off)>(mi, addr + offsetof(Elf_W(Phdr), p_offset), &phdr.p_offset)) {
+      return;
+    }
+    if (phdr.p_type == PT_LOAD && phdr.p_offset == mi->offset) {
+      if (!get_val<Elf_W(Addr)>(mi, addr + offsetof(Elf_W(Phdr), p_vaddr), &phdr.p_vaddr)) {
+        return;
+      }
+      mi->load_base = phdr.p_vaddr;
+      return;
+    }
+    addr += sizeof(phdr);
+  }
+}
+
 // Find the containing map info for the PC.
 __LIBC_HIDDEN__ const mapinfo_t* mapinfo_find(mapinfo_t* mi, uintptr_t pc, uintptr_t* rel_pc) {
   for (; mi != NULL; mi = mi->next) {
     if ((pc >= mi->start) && (pc < mi->end)) {
-      *rel_pc = pc - mi->start;
+      if (!mi->load_base_read) {
+        mapinfo_read_loadbase(mi);
+      }
+      *rel_pc = pc - mi->start + mi->load_base;
       return mi;
     }
   }