Use VDSO for clock_gettime(2) and gettimeofday(2).

Bug: 15387103

(cherry picked from commit 625993dfbb)

Change-Id: I0e156d7049ba1495902259071a96936592e74025

benchmarks/Android.mk

@@ -54,7 +54,7 @@ LOCAL_SHARED_LIBRARIES += libstlport
 LOCAL_SRC_FILES := $(benchmark_src_files)
 include $(BUILD_EXECUTABLE)
 
-ifeq ($(HOST_OS)-$(HOST_ARCH),linux-x86)
+ifeq ($(HOST_OS)-$(HOST_ARCH),$(filter $(HOST_OS)-$(HOST_ARCH),linux-x86 linux-x86_64))
 ifeq ($(TARGET_ARCH),x86)
 LINKER = linker
 NATIVE_SUFFIX=32

benchmarks/time_benchmark.cpp

@@ -16,6 +16,7 @@
 
 #include "benchmark.h"
 
+#include <sys/syscall.h>
 #include <time.h>
 
 #if defined(__BIONIC__)
@@ -41,7 +42,7 @@ BENCHMARK(BM_time_localtime_tz);
 static void BM_time_clock_gettime(int iters) {
   StartBenchmarkTiming();
 
-  struct timespec t;
+  timespec t;
   for (int i = 0; i < iters; ++i) {
     clock_gettime(CLOCK_MONOTONIC, &t);
   }
@@ -49,3 +50,50 @@ static void BM_time_clock_gettime(int iters) {
   StopBenchmarkTiming();
 }
 BENCHMARK(BM_time_clock_gettime);
+
+static void BM_time_clock_gettime_syscall(int iters) {
+  StartBenchmarkTiming();
+
+  timespec t;
+  for (int i = 0; i < iters; ++i) {
+    syscall(__NR_clock_gettime, CLOCK_MONOTONIC, &t);
+  }
+
+  StopBenchmarkTiming();
+}
+BENCHMARK(BM_time_clock_gettime_syscall);
+
+static void BM_time_gettimeofday(int iters) {
+  StartBenchmarkTiming();
+
+  timeval tv;
+  for (int i = 0; i < iters; ++i) {
+    gettimeofday(&tv, NULL);
+  }
+
+  StopBenchmarkTiming();
+}
+BENCHMARK(BM_time_gettimeofday);
+
+static void BM_time_gettimeofday_syscall(int iters) {
+  StartBenchmarkTiming();
+
+  timeval tv;
+  for (int i = 0; i < iters; ++i) {
+    syscall(__NR_gettimeofday, &tv, NULL);
+  }
+
+  StopBenchmarkTiming();
+}
+BENCHMARK(BM_time_gettimeofday_syscall);
+
+static void BM_time_time(int iters) {
+  StartBenchmarkTiming();
+
+  for (int i = 0; i < iters; ++i) {
+    time(NULL);
+  }
+
+  StopBenchmarkTiming();
+}
+BENCHMARK(BM_time_time);

libc/Android.mk

@@ -229,6 +229,7 @@ libc_bionic_src_files := \
     bionic/umount.cpp \
     bionic/unlink.cpp \
     bionic/utimes.cpp \
+    bionic/vdso.cpp \
     bionic/vfork.cpp \
     bionic/wait.cpp \
     bionic/wchar.cpp \

libc/SYSCALLS.TXT

@@ -194,11 +194,9 @@ int     swapon(const char*, int) all
 int     swapoff(const char*) all
 
 # time
-int           gettimeofday(struct timeval*, struct timezone*)       all
 int           settimeofday(const struct timeval*, const struct timezone*)   all
 clock_t       times(struct tms*)       all
 int           nanosleep(const struct timespec*, struct timespec*)   all
-int           clock_gettime(clockid_t clk_id, struct timespec* tp)    all
 int           clock_settime(clockid_t clk_id, const struct timespec* tp)  all
 int           clock_getres(clockid_t clk_id, struct timespec* res)   all
 int           clock_nanosleep(clockid_t clock_id, int flags, const struct timespec* req, struct timespec* rem)  all
@@ -320,3 +318,9 @@ int     __set_tls:set_thread_area(void*) mips,mips64
 
 # x86-specific
 int     __set_thread_area:set_thread_area(void*) x86
+
+# vdso stuff.
+int clock_gettime(clockid_t, timespec*)                 arm,mips,mips64,x86
+int __clock_gettime:clock_gettime(clockid_t, timespec*) arm64,x86_64
+int gettimeofday(timeval*, timezone*)                   arm,mips,mips64,x86
+int __gettimeofday:gettimeofday(timeval*, timezone*)    arm64,x86_64

libc/arch-arm64/syscalls/__clock_gettime.S

@@ -2,7 +2,7 @@
 
 #include <private/bionic_asm.h>
 
-ENTRY(clock_gettime)
+ENTRY(__clock_gettime)
     mov     x8, __NR_clock_gettime
     svc     #0
 
@@ -11,4 +11,5 @@ ENTRY(clock_gettime)
     b.hi    __set_errno
 
     ret
-END(clock_gettime)
+END(__clock_gettime)
+.hidden __clock_gettime

libc/arch-arm64/syscalls/__gettimeofday.S

@@ -2,7 +2,7 @@
 
 #include <private/bionic_asm.h>
 
-ENTRY(gettimeofday)
+ENTRY(__gettimeofday)
     mov     x8, __NR_gettimeofday
     svc     #0
 
@@ -11,4 +11,5 @@ ENTRY(gettimeofday)
     b.hi    __set_errno
 
     ret
-END(gettimeofday)
+END(__gettimeofday)
+.hidden __gettimeofday

libc/arch-x86_64/syscalls/__clock_gettime.S

@@ -2,7 +2,7 @@
 
 #include <private/bionic_asm.h>
 
-ENTRY(clock_gettime)
+ENTRY(__clock_gettime)
     movl    $__NR_clock_gettime, %eax
     syscall
     cmpq    $-MAX_ERRNO, %rax
@@ -12,4 +12,5 @@ ENTRY(clock_gettime)
     call    __set_errno
 1:
     ret
-END(clock_gettime)
+END(__clock_gettime)
+.hidden __clock_gettime

libc/arch-x86_64/syscalls/__gettimeofday.S

@@ -2,7 +2,7 @@
 
 #include <private/bionic_asm.h>
 
-ENTRY(gettimeofday)
+ENTRY(__gettimeofday)
     movl    $__NR_gettimeofday, %eax
     syscall
     cmpq    $-MAX_ERRNO, %rax
@@ -12,4 +12,5 @@ ENTRY(gettimeofday)
     call    __set_errno
 1:
     ret
-END(gettimeofday)
+END(__gettimeofday)
+.hidden __gettimeofday

libc/bionic/dl_iterate_phdr_static.cpp

@@ -27,6 +27,7 @@
  */
 
 #include <elf.h>
+#include <string.h>
 #include <sys/auxv.h>
 #include <sys/types.h>
 #include <link.h>
@@ -37,11 +38,9 @@ extern "C" void* __executable_start;
 int dl_iterate_phdr(int (*cb)(struct dl_phdr_info* info, size_t size, void* data), void* data) {
   ElfW(Ehdr)* ehdr = reinterpret_cast<ElfW(Ehdr)*>(&__executable_start);
 
-  // TODO: again, copied from linker.c. Find a better home for this later.
-  if (ehdr->e_ident[EI_MAG0] != ELFMAG0) return -1;
-  if (ehdr->e_ident[EI_MAG1] != ELFMAG1) return -1;
-  if (ehdr->e_ident[EI_MAG2] != ELFMAG2) return -1;
-  if (ehdr->e_ident[EI_MAG3] != ELFMAG3) return -1;
+  if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG) != 0) {
+    return -1;
+  }
 
   // Dynamic binaries get their dl_iterate_phdr from the dynamic linker, but
   // static binaries get this. We don't have a list of shared objects to
@@ -54,7 +53,7 @@ int dl_iterate_phdr(int (*cb)(struct dl_phdr_info* info, size_t size, void* data
   exe_info.dlpi_phdr = reinterpret_cast<ElfW(Phdr)*>(reinterpret_cast<uintptr_t>(ehdr) + ehdr->e_phoff);
   exe_info.dlpi_phnum = ehdr->e_phnum;
 
-#ifdef AT_SYSINFO_EHDR
+#if defined(AT_SYSINFO_EHDR)
   // Try the executable first.
   int rc = cb(&exe_info, sizeof(exe_info), data);
   if (rc != 0) {

libc/bionic/libc_init_common.cpp

@@ -51,6 +51,8 @@ extern "C" int __system_properties_init(void);
 extern "C" int __set_tls(void* ptr);
 extern "C" int __set_tid_address(int* tid_address);
 
+void __libc_init_vdso();
+
 // Not public, but well-known in the BSDs.
 const char* __progname;
 
@@ -129,6 +131,8 @@ void __libc_init_common(KernelArgumentBlock& args) {
   _pthread_internal_add(main_thread);
 
   __system_properties_init(); // Requires 'environ'.
+
+  __libc_init_vdso();
 }
 
 /* This function will be called during normal program termination

libc/bionic/vdso.cpp

@@ -0,0 +1,129 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <link.h>
+#include <sys/auxv.h>
+#include <unistd.h>
+
+// x86 has a vdso, but there's nothing useful to us in it.
+#if defined(__aarch64__) || defined(__x86_64__)
+
+#if defined(__aarch64__)
+#define VDSO_CLOCK_GETTIME_SYMBOL "__kernel_clock_gettime"
+#define VDSO_GETTIMEOFDAY_SYMBOL  "__kernel_gettimeofday"
+#elif defined(__x86_64__)
+#define VDSO_CLOCK_GETTIME_SYMBOL "__vdso_clock_gettime"
+#define VDSO_GETTIMEOFDAY_SYMBOL  "__vdso_gettimeofday"
+#endif
+
+#include <time.h>
+
+extern "C" int __clock_gettime(int, timespec*);
+extern "C" int __gettimeofday(timeval*, struct timezone*);
+
+struct vdso_entry {
+  const char* name;
+  void* fn;
+};
+
+enum {
+  VDSO_CLOCK_GETTIME = 0,
+  VDSO_GETTIMEOFDAY,
+  VDSO_END
+};
+
+static vdso_entry vdso_entries[] = {
+  [VDSO_CLOCK_GETTIME] = { VDSO_CLOCK_GETTIME_SYMBOL, reinterpret_cast<void*>(__clock_gettime) },
+  [VDSO_GETTIMEOFDAY] = { VDSO_GETTIMEOFDAY_SYMBOL, reinterpret_cast<void*>(__gettimeofday) },
+};
+
+int clock_gettime(int clock_id, timespec* tp) {
+  static int (*vdso_clock_gettime)(int, timespec*) =
+      (int (*)(int, timespec*)) vdso_entries[VDSO_CLOCK_GETTIME].fn;
+  return vdso_clock_gettime(clock_id, tp);
+}
+
+int gettimeofday(timeval* tv, struct timezone* tz) {
+  static int (*vdso_gettimeofday)(timeval*, struct timezone*) =
+      (int (*)(timeval*, struct timezone*)) vdso_entries[VDSO_GETTIMEOFDAY].fn;
+  return vdso_gettimeofday(tv, tz);
+}
+
+void __libc_init_vdso() {
+  // Do we have a vdso?
+  uintptr_t vdso_ehdr_addr = getauxval(AT_SYSINFO_EHDR);
+  ElfW(Ehdr)* vdso_ehdr = reinterpret_cast<ElfW(Ehdr)*>(vdso_ehdr_addr);
+  if (vdso_ehdr == NULL) {
+    return;
+  }
+
+  // How many symbols does it have?
+  size_t symbol_count = 0;
+  ElfW(Shdr)* vdso_shdr = reinterpret_cast<ElfW(Shdr)*>(vdso_ehdr_addr + vdso_ehdr->e_shoff);
+  for (size_t i = 0; i < vdso_ehdr->e_shnum; ++i) {
+    if (vdso_shdr[i].sh_type == SHT_DYNSYM) {
+      symbol_count = vdso_shdr[i].sh_size / sizeof(ElfW(Sym));
+    }
+  }
+  if (symbol_count == 0) {
+    return;
+  }
+
+  // Where's the dynamic table?
+  ElfW(Addr) vdso_addr = 0;
+  ElfW(Dyn)* vdso_dyn = NULL;
+  ElfW(Phdr)* vdso_phdr = reinterpret_cast<ElfW(Phdr)*>(vdso_ehdr_addr + vdso_ehdr->e_phoff);
+  for (size_t i = 0; i < vdso_ehdr->e_phnum; ++i) {
+    if (vdso_phdr[i].p_type == PT_DYNAMIC) {
+      vdso_dyn = reinterpret_cast<ElfW(Dyn)*>(vdso_ehdr_addr + vdso_phdr[i].p_offset);
+    } else if (vdso_phdr[i].p_type == PT_LOAD) {
+      vdso_addr = vdso_ehdr_addr + vdso_phdr[i].p_offset - vdso_phdr[i].p_vaddr;
+    }
+  }
+  if (vdso_addr == 0 || vdso_dyn == NULL) {
+    return;
+  }
+
+  // Where are the string and symbol tables?
+  const char* strtab = NULL;
+  ElfW(Sym)* symtab = NULL;
+  for (ElfW(Dyn)* d = vdso_dyn; d->d_tag != DT_NULL; ++d) {
+    if (d->d_tag == DT_STRTAB) {
+      strtab = reinterpret_cast<const char*>(vdso_addr + d->d_un.d_ptr);
+    } else if (d->d_tag == DT_SYMTAB) {
+      symtab = reinterpret_cast<ElfW(Sym)*>(vdso_addr + d->d_un.d_ptr);
+    }
+  }
+  if (strtab == NULL || symtab == NULL) {
+    return;
+  }
+
+  // Are there any symbols we want?
+  for (size_t i = 0; i < symbol_count; ++i) {
+    for (size_t j = 0; j < VDSO_END; ++j) {
+      if (strcmp(vdso_entries[j].name, strtab + symtab[i].st_name) == 0) {
+        vdso_entries[j].fn = reinterpret_cast<void*>(vdso_addr + symtab[i].st_value);
+      }
+    }
+  }
+}
+
+#else
+
+void __libc_init_vdso() {
+}
+
+#endif

linker/linker_phdr.cpp

@@ -156,10 +156,7 @@ bool ElfReader::ReadElfHeader() {
 }
 
 bool ElfReader::VerifyElfHeader() {
-  if (header_.e_ident[EI_MAG0] != ELFMAG0 ||
-      header_.e_ident[EI_MAG1] != ELFMAG1 ||
-      header_.e_ident[EI_MAG2] != ELFMAG2 ||
-      header_.e_ident[EI_MAG3] != ELFMAG3) {
+  if (memcmp(header_.e_ident, ELFMAG, SELFMAG) != 0) {
     DL_ERR("\"%s\" has bad ELF magic", name_);
     return false;
   }

tests/sys_time_test.cpp

@@ -17,6 +17,7 @@
 #include <gtest/gtest.h>
 
 #include <errno.h>
+#include <sys/syscall.h>
 #include <sys/time.h>
 
 #include "TemporaryFile.h"
@@ -46,3 +47,23 @@ TEST(sys_time, utimes_NULL) {
   TemporaryFile tf;
   ASSERT_EQ(0, utimes(tf.filename, NULL));
 }
+
+TEST(sys_time, gettimeofday) {
+  // Try to ensure that our vdso gettimeofday is working.
+  timeval tv1;
+  ASSERT_EQ(0, gettimeofday(&tv1, NULL));
+  timeval tv2;
+  ASSERT_EQ(0, syscall(__NR_gettimeofday, &tv2, NULL));
+
+  // What's the difference between the two?
+  tv2.tv_sec -= tv1.tv_sec;
+  tv2.tv_usec -= tv1.tv_usec;
+  if (tv2.tv_usec < 0) {
+    --tv2.tv_sec;
+    tv2.tv_usec += 1000000;
+  }
+
+  // Should be less than (a very generous, to try to avoid flakiness) 1000us.
+  ASSERT_EQ(0, tv2.tv_sec);
+  ASSERT_LT(tv2.tv_usec, 1000);
+}

tests/time_test.cpp

@@ -21,6 +21,7 @@
 #include <gtest/gtest.h>
 #include <pthread.h>
 #include <signal.h>
+#include <sys/syscall.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 
@@ -419,3 +420,23 @@ TEST(time, timer_delete_from_timer_thread) {
   ASSERT_EQ(ESRCH, pthread_detach(tdd.thread_id));
 #endif
 }
+
+TEST(time, clock_gettime) {
+  // Try to ensure that our vdso clock_gettime is working.
+  timespec ts1;
+  ASSERT_EQ(0, clock_gettime(CLOCK_MONOTONIC, &ts1));
+  timespec ts2;
+  ASSERT_EQ(0, syscall(__NR_clock_gettime, CLOCK_MONOTONIC, &ts2));
+
+  // What's the difference between the two?
+  ts2.tv_sec -= ts1.tv_sec;
+  ts2.tv_nsec -= ts1.tv_nsec;
+  if (ts2.tv_nsec < 0) {
+    --ts2.tv_sec;
+    ts2.tv_nsec += 1000000000;
+  }
+
+  // Should be less than (a very generous, to try to avoid flakiness) 1000000ns.
+  ASSERT_EQ(0, ts2.tv_sec);
+  ASSERT_LT(ts2.tv_nsec, 1000000);
+}