Roll gtest-parallel.

Includes modification by kwiberg@ for starting slowest tests first,
reducing total runtime without increasing workers.

BUG=
R=kwiberg@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/46339004

Cr-Commit-Position: refs/heads/master@{#9207}

third_party/gtest-parallel/README.webrtc

@@ -1,5 +1,5 @@
 URL: https://github.com/google/gtest-parallel
-Version: 48e584a52bb9db1d1c915ea33463e9e4e1b36d1b
+Version: 3405a00ea6661d39f416faf7ccddf3c05fbfe19c
 License: Apache 2.0
 License File: LICENSE
 

third_party/gtest-parallel/gtest-parallel

@@ -12,11 +12,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import cPickle
+import gzip
+import multiprocessing
 import optparse
+import os
 import subprocess
 import sys
 import threading
 import time
+import zlib
 
 stdout_lock = threading.Lock()
 class FilterFormat:
@@ -82,6 +87,52 @@ class RawFormat:
   def end(self):
     pass
 
+# Record of test runtimes. Has built-in locking.
+class TestTimes(object):
+  def __init__(self, save_file):
+    "Create new object seeded with saved test times from the given file."
+    self.__times = {}  # (test binary, test name) -> runtime in ms
+
+    # Protects calls to record_test_time(); other calls are not
+    # expected to be made concurrently.
+    self.__lock = threading.Lock()
+
+    try:
+      with gzip.GzipFile(save_file, "rb") as f:
+        times = cPickle.load(f)
+    except (EOFError, IOError, cPickle.UnpicklingError, zlib.error):
+      # File doesn't exist, isn't readable, is malformed---whatever.
+      # Just ignore it.
+      return
+
+    # Discard saved times if the format isn't right.
+    if type(times) is not dict:
+      return
+    for ((test_binary, test_name), runtime) in times.items():
+      if (type(test_binary) is not str or type(test_name) is not str
+          or type(runtime) not in {int, long}):
+        return
+
+    self.__times = times
+
+  def get_test_time(self, binary, testname):
+    "Return the last duration for the given test, or 0 if there's no record."
+    return self.__times.get((binary, testname), 0)
+
+  def record_test_time(self, binary, testname, runtime_ms):
+    "Record that the given test ran in the specified number of milliseconds."
+    with self.__lock:
+      self.__times[(binary, testname)] = runtime_ms
+
+  def write_to_file(self, save_file):
+    "Write all the times to file."
+    try:
+      with open(save_file, "wb") as f:
+        with gzip.GzipFile("", "wb", 9, f) as gzf:
+          cPickle.dump(self.__times, gzf, cPickle.HIGHEST_PROTOCOL)
+    except IOError:
+      pass  # ignore errors---saving the times isn't that important
+
 # Remove additional arguments (anything after --).
 additional_args = []
 
@@ -96,7 +147,8 @@ parser = optparse.OptionParser(
 
 parser.add_option('-r', '--repeat', type='int', default=1,
                   help='repeat tests')
-parser.add_option('-w', '--workers', type='int', default=16,
+parser.add_option('-w', '--workers', type='int',
+                  default=multiprocessing.cpu_count(),
                   help='number of workers to spawn')
 parser.add_option('--gtest_color', type='string', default='yes',
                   help='color output')
@@ -122,6 +174,8 @@ else:
   sys.exit("Unknown output format: " + options.format)
 
 # Find tests.
+save_file = os.path.join(os.path.expanduser("~"), ".gtest-parallel-times")
+times = TestTimes(save_file)
 tests = []
 for test_binary in binaries:
   command = [test_binary]
@@ -132,8 +186,11 @@ for test_binary in binaries:
   if options.gtest_filter != '':
     list_command += ['--gtest_filter=' + options.gtest_filter]
 
-  test_list = subprocess.Popen(list_command + ['--gtest_list_tests'],
+  try:
-                               stdout=subprocess.PIPE).communicate()[0]
+    test_list = subprocess.Popen(list_command + ['--gtest_list_tests'],
+                                 stdout=subprocess.PIPE).communicate()[0]
+  except OSError as e:
+    sys.exit("%s: %s" % (test_binary, str(e)))
 
   command += additional_args
 
@@ -152,7 +209,9 @@ for test_binary in binaries:
       continue
 
     test = test_group + line
-    tests.append((test_binary, command, test))
+    tests.append((times.get_test_time(test_binary, test),
+                  test_binary, test, command))
+tests.sort(reverse=True)
 
 # Repeat tests (-r flag).
 tests *= options.repeat
@@ -161,6 +220,8 @@ job_id = 0
 logger.log(str(-1) + ': TESTCNT ' + ' ' + str(len(tests)))
 
 exit_code = 0
+
+# Run the specified job. Returns the elapsed time in milliseconds.
 def run_job((command, job_id, test)):
   begin = time.time()
   sub = subprocess.Popen(command + ['--gtest_filter=' + test] +
@@ -176,10 +237,11 @@ def run_job((command, job_id, test)):
 
   code = sub.wait()
   runtime_ms = int(1000 * (time.time() - begin))
-  logger.log(str(job_id) + ': EXIT ' + str(code) + ' ' + str(runtime_ms))
+  logger.log("%s: EXIT %s %d" % (job_id, code, runtime_ms))
   if code != 0:
     global exit_code
     exit_code = code
+  return runtime_ms
 
 def worker():
   global job_id
@@ -187,14 +249,14 @@ def worker():
     job = None
     test_lock.acquire()
     if job_id < len(tests):
-      (test_binary, command, test) = tests[job_id]
+      (_, test_binary, test, command) = tests[job_id]
       logger.log(str(job_id) + ': TEST ' + test_binary + ' ' + test)
       job = (command, job_id, test)
     job_id += 1
     test_lock.release()
     if job is None:
       return
-    run_job(job)
+    times.record_test_time(test_binary, test, run_job(job))
 
 def start_daemon(func):
   t = threading.Thread(target=func)
@@ -206,4 +268,5 @@ workers = [start_daemon(worker) for i in range(options.workers)]
 
 [t.join() for t in workers]
 logger.end()
+times.write_to_file(save_file)
 sys.exit(exit_code)