Add remote testing support to the lit config.

Executors can be specified at configure time by using the -DLIBCXX_EXECUTOR=""
option. Examples include:

  $ cmake <other_flags> -DLIBCXX_EXECUTOR="TimeoutExecutor(30,LocalExecutor())"
      This runs individual tests with a maximum duration

  $ cmake <other_flags> -DLIBCXX_EXECUTOR="SSHExecutor('hostname','username')"
      This runs tests on a remote target, using scp to shuttle binaries to the
      target, and ssh to invoke commands there.

  $ cmake <other_flags> -DLIBCXX_EXECUTOR="PrefixExecutor('/path/to/run/script',LocalExecutor())"
      This assumes the script knows how to copy run the executables passed to it,
      and allows for the ultimate control. This is useful for running things
      inside emulators like Valgrind & QEMU.

TODO: This doesn't claim to support ShTest tests yet, that will take a bit more
  thought & finagling (I'm still not sure how to orchestrate copy-in for those cases.

  I've also punted on what to do about tests that read data files. The testsuite
  has several tests that need to read *.dat files placed next to them, and
  currently those aren't copied over when using, say, an SSHExecutor. The
  affected tests are:

     libc++ :: std/input.output/file.streams/fstreams/filebuf.virtuals/pbackfail.pass.cpp
     libc++ :: std/input.output/file.streams/fstreams/filebuf.virtuals/underflow.pass.cpp
     libc++ :: std/input.output/file.streams/fstreams/ifstream.assign/member_swap.pass.cpp
     libc++ :: std/input.output/file.streams/fstreams/ifstream.assign/move_assign.pass.cpp
     libc++ :: std/input.output/file.streams/fstreams/ifstream.assign/nonmember_swap.pass.cpp
     libc++ :: std/input.output/file.streams/fstreams/ifstream.cons/move.pass.cpp
     libc++ :: std/input.output/file.streams/fstreams/ifstream.cons/pointer.pass.cpp
     libc++ :: std/input.output/file.streams/fstreams/ifstream.cons/string.pass.cpp
     libc++ :: std/input.output/file.streams/fstreams/ifstream.members/close.pass.cpp
     libc++ :: std/input.output/file.streams/fstreams/ifstream.members/open_pointer.pass.cpp
     libc++ :: std/input.output/file.streams/fstreams/ifstream.members/open_string.pass.cpp
     libc++ :: std/input.output/file.streams/fstreams/ifstream.members/rdbuf.pass.cpp
     libc++ :: std/localization/locales/locale.convenience/conversions/conversions.buffer/pbackfail.pass.cpp
     libc++ :: std/localization/locales/locale.convenience/conversions/conversions.buffer/underflow.pass.cpp

Note: One thing to watch out for when using the SSHExecutor for cross-testing is
  that you'll also want to specify a TargetInfo object (so that the host's
  features aren't used for available-features checks and flags setup).

http://reviews.llvm.org/D7380


git-svn-id: https://llvm.org/svn/llvm-project/libcxx/trunk@230592 91177308-0d34-0410-b5e6-96231b3b80d8

test/CMakeLists.txt

@@ -46,6 +46,8 @@ if (LIT_EXECUTABLE)
   pythonize_bool(LIBCXX_ENABLE_MONOTONIC_CLOCK)
   set(LIBCXX_TARGET_INFO "libcxx.test.target_info.LocalTI" CACHE STRING
       "TargetInfo to use when setting up test environment.")
+  set(LIBCXX_EXECUTOR "None" CACHE STRING
+      "Executor to use when running tests.")
 
   set(AUTO_GEN_COMMENT "## Autogenerated by libcxx configuration.\n# Do not edit!")
 

test/libcxx/test/config.py

@@ -12,7 +12,8 @@ import lit.util  # pylint: disable=import-error,no-name-in-module
 
 from libcxx.test.format import LibcxxTestFormat
 from libcxx.compiler import CXXCompiler
-
+from libcxx.test.executor import *
+from libcxx.test.tracing import *
 
 def loadSiteConfig(lit_config, config, param_name, env_name):
     # We haven't loaded the site specific configuration (the user is
@@ -78,6 +79,7 @@ class Configuration(object):
             "parameter '{}' should be true or false".format(name))
 
     def configure(self):
+        self.configure_executor()
         self.configure_target_info()
         self.configure_cxx()
         self.configure_triple()
@@ -110,6 +112,32 @@ class Configuration(object):
                              list(self.config.available_features))
         self.lit_config.note('Using environment: %r' % self.env)
 
+    def get_test_format(self):
+        return LibcxxTestFormat(
+            self.cxx,
+            self.use_clang_verify,
+            self.execute_external,
+            self.executor,
+            exec_env=self.env)
+
+    def configure_executor(self):
+        exec_str = self.get_lit_conf('executor', "None")
+        te = eval(exec_str)
+        if te:
+            self.lit_config.note("Using executor: %r" % exec_str)
+            if self.lit_config.useValgrind:
+                # We have no way of knowing where in the chain the
+                # ValgrindExecutor is supposed to go. It is likely
+                # that the user wants it at the end, but we have no
+                # way of getting at that easily.
+                selt.lit_config.fatal("Cannot infer how to create a Valgrind "
+                                      " executor.")
+        else:
+            te = LocalExecutor()
+            if self.lit_config.useValgrind:
+                te = ValgrindExecutor(self.lit_config.valgrindArgs, te)
+        self.executor = te
+
     def configure_target_info(self):
         default = "libcxx.test.target_info.LocalTI"
         info_str = self.get_lit_conf('target_info', default)
@@ -119,13 +147,6 @@ class Configuration(object):
         if info_str != default:
             self.lit_config.note("inferred target_info as: %r" % info_str)
 
-    def get_test_format(self):
-        return LibcxxTestFormat(
-            self.cxx,
-            self.use_clang_verify,
-            self.execute_external,
-            exec_env=self.env)
-
     def configure_cxx(self):
         # Gather various compiler parameters.
         cxx = self.get_lit_conf('cxx_under_test')
@@ -319,7 +340,7 @@ class Configuration(object):
         # Configure include paths
         self.cxx.compile_flags += ['-nostdinc++']
         self.configure_compile_flags_header_includes()
-        if sys.platform.startswith('linux'):
+        if self.target_info.platform() == 'linux':
             self.cxx.compile_flags += ['-D__STDC_FORMAT_MACROS',
                                        '-D__STDC_LIMIT_MACROS',
                                        '-D__STDC_CONSTANT_MACROS']
@@ -601,15 +622,16 @@ class Configuration(object):
             # linux-gnu is needed in the triple to properly identify linuxes
             # that use GLIBC. Handle redhat and opensuse triples as special
             # cases and append the missing `-gnu` portion.
-            if target_triple.endswith('redhat-linux') or \
+            if (target_triple.endswith('redhat-linux') or
-               target_triple.endswith('suse-linux'):
+                target_triple.endswith('suse-linux')):
                 target_triple += '-gnu'
             self.config.target_triple = target_triple
             self.lit_config.note(
                 "inferred target_triple as: %r" % self.config.target_triple)
 
     def configure_env(self):
-        if sys.platform == 'darwin' and not self.use_system_cxx_lib:
+        if (self.target_info.platform() == 'darwin' and
+            not self.use_system_cxx_lib):
             libcxx_library = self.get_lit_conf('libcxx_library')
             if libcxx_library:
                 cxx_library_root = os.path.dirname(libcxx_library)

test/libcxx/test/executor.py

@@ -0,0 +1,166 @@
+import os
+
+import tracing
+
+from lit.util import executeCommand  # pylint: disable=import-error
+
+
+class Executor(object):
+    def run(self, exe_path, cmd, local_cwd, env=None):
+        """Execute a command.
+            Be very careful not to change shared state in this function.
+            Executor objects are shared between python processes in `lit -jN`.
+        Args:
+            exe_path: str:   Local path to the executable to be run
+            cmd: [str]:      subprocess.call style command
+            local_cwd: str:  Local path to the working directory
+            env: {str: str}: Environment variables to execute under
+        Returns:
+            out, err, exitCode
+        """
+        raise NotImplementedError
+
+
+class LocalExecutor(Executor):
+    def __init__(self):
+        super(LocalExecutor, self).__init__()
+
+    def run(self, exe_path, cmd=None, work_dir='.', env=None):
+        cmd = cmd or [exe_path]
+        env_cmd = []
+        if env:
+            env_cmd += ['env']
+            env_cmd += ['%s=%s' % (k, v) for k, v in env.items()]
+        if work_dir == '.':
+            work_dir = os.getcwd()
+        return executeCommand(env_cmd + cmd, cwd=work_dir)
+
+
+class PrefixExecutor(Executor):
+    """Prefix an executor with some other command wrapper.
+
+    Most useful for setting ulimits on commands, or running an emulator like
+    qemu and valgrind.
+    """
+    def __init__(self, commandPrefix, chain):
+        super(PrefixExecutor, self).__init__()
+
+        self.commandPrefix = commandPrefix
+        self.chain = chain
+
+    def run(self, exe_path, cmd=None, work_dir='.', env=None):
+        cmd = cmd or [exe_path]
+        return self.chain.run(self.commandPrefix + cmd, work_dir, env=env)
+
+
+class PostfixExecutor(Executor):
+    """Postfix an executor with some args."""
+    def __init__(self, commandPostfix, chain):
+        super(PostfixExecutor, self).__init__()
+
+        self.commandPostfix = commandPostfix
+        self.chain = chain
+
+    def run(self, exe_path, cmd=None, work_dir='.', env=None):
+        cmd = cmd or [exe_path]
+        return self.chain.run(cmd + self.commandPostfix, work_dir, env=env)
+
+
+
+class TimeoutExecutor(PrefixExecutor):
+    """Execute another action under a timeout.
+
+    Deprecated. http://reviews.llvm.org/D6584 adds timeouts to LIT.
+    """
+    def __init__(self, duration, chain):
+        super(TimeoutExecutor, self).__init__(
+            ['timeout', duration], chain)
+
+
+class SSHExecutor(Executor):
+    def __init__(self, host, username=None):
+        super(SSHExecutor, self).__init__()
+
+        self.user_prefix = username + '@' if username else ''
+        self.host = host
+        self.scp_command = 'scp'
+        self.ssh_command = 'ssh'
+
+        self.local_run = executeCommand
+        # TODO(jroelofs): switch this on some -super-verbose-debug config flag
+        if False:
+            self.local_run = tracing.trace_function(
+                self.local_run, log_calls=True, log_results=True,
+                label='ssh_local')
+
+    def remote_temp_dir(self):
+        return self._remote_temp(True)
+
+    def remote_temp_file(self):
+        return self._remote_temp(False)
+
+    def _remote_temp(self, is_dir):
+        # TODO: detect what the target system is, and use the correct
+        # mktemp command for it. (linux and darwin differ here, and I'm
+        # sure windows has another way to do it)
+
+        # Not sure how to do suffix on osx yet
+        dir_arg = '-d' if is_dir else ''
+        cmd = 'mktemp -q {} /tmp/libcxx.XXXXXXXXXX'.format(dir_arg)
+        temp_path, err, exitCode = self.__execute_command_remote([cmd])
+        temp_path = temp_path.strip()
+        if exitCode != 0:
+            raise RuntimeError(err)
+        return temp_path
+
+    def copy_in(self, local_srcs, remote_dsts):
+        scp = self.scp_command
+        remote = self.host
+        remote = self.user_prefix + remote
+
+        # This could be wrapped up in a tar->scp->untar for performance
+        # if there are lots of files to be copied/moved
+        for src, dst in zip(local_srcs, remote_dsts):
+            cmd = [scp, '-p', src, remote + ':' + dst]
+            self.local_run(cmd)
+
+    def delete_remote(self, remote):
+        try:
+            self.__execute_command_remote(['rm', '-rf', remote])
+        except OSError:
+            # TODO: Log failure to delete?
+            pass
+
+    def run(self, exe_path, cmd=None, work_dir='.', env=None):
+        target_exe_path = None
+        target_cwd = None
+        try:
+            target_exe_path = self.remote_temp_file()
+            target_cwd = self.remote_temp_dir()
+            if cmd:
+                # Replace exe_path with target_exe_path.
+                cmd = [c if c != exe_path else target_exe_path for c in cmd]
+            else:
+                cmd = [target_exe_path]
+            self.copy_in([exe_path], [target_exe_path])
+            return self.__execute_command_remote(cmd, target_cwd, env)
+        except:
+            raise
+        finally:
+            if target_exe_path:
+                self.delete_remote(target_exe_path)
+            if target_cwd:
+                self.delete_remote(target_cwd)
+
+    def __execute_command_remote(self, cmd, remote_work_dir='.', env=None):
+        remote = self.user_prefix + self.host
+        ssh_cmd = [self.ssh_command, '-oBatchMode=yes', remote]
+        if env:
+            env_cmd = ['env'] + ['%s=%s' % (k, v) for k, v in env.items()]
+        else:
+            env_cmd = []
+        remote_cmd = ' '.join(env_cmd + cmd)
+        if remote_work_dir != '.':
+            remote_cmd = 'cd ' + remote_work_dir + ' && ' + remote_cmd
+        return self.local_run(ssh_cmd + [remote_cmd])
+

test/libcxx/test/format.py

@@ -20,10 +20,12 @@ class LibcxxTestFormat(object):
       FOO.sh.cpp   - A test that uses LIT's ShTest format.
     """
 
-    def __init__(self, cxx, use_verify_for_fail, execute_external, exec_env):
+    def __init__(self, cxx, use_verify_for_fail, execute_external,
+                 executor, exec_env):
         self.cxx = cxx
         self.use_verify_for_fail = use_verify_for_fail
         self.execute_external = execute_external
+        self.executor = executor
         self.exec_env = dict(exec_env)
 
     # TODO: Move this into lit's FileBasedTest
@@ -73,6 +75,10 @@ class LibcxxTestFormat(object):
 
         # Dispatch the test based on its suffix.
         if is_sh_test:
+            if self.executor:
+                # We can't run ShTest tests with a executor yet.
+                # For now, bail on trying to run them
+                return lit.Test.UNSUPPORTED, 'ShTest format not yet supported'
             return lit.TestRunner._runShTest(test, lit_config,
                                              self.execute_external, script,
                                              tmpBase, execDir)
@@ -104,15 +110,12 @@ class LibcxxTestFormat(object):
                 report += "Compilation failed unexpectedly!"
                 return lit.Test.FAIL, report
             # Run the test
-            cmd = []
+            local_cwd = os.path.dirname(source_path)
+            env = None
             if self.exec_env:
-                cmd += ['env']
+                env = self.exec_env
-                cmd += ['%s=%s' % (k, v) for k, v in self.exec_env.items()]
+            out, err, rc = self.executor.run(exec_path, [exec_path],
-            if lit_config.useValgrind:
+                                             local_cwd, env)
-                cmd = lit_config.valgrindArgs + cmd
-            cmd += [exec_path]
-            out, err, rc = lit.util.executeCommand(
-                cmd, cwd=os.path.dirname(source_path))
             if rc != 0:
                 report = libcxx.util.makeReport(cmd, out, err, rc)
                 report = "Compiled With: %s\n%s" % (compile_cmd, report)

test/libcxx/test/tracing.py

@@ -0,0 +1,34 @@
+import os
+import inspect
+
+
+def trace_function(function, log_calls, log_results, label=''):
+    def wrapper(*args, **kwargs):
+        kwarg_strs = ['{}={}'.format(k, v) for (k, v) in kwargs]
+        arg_str = ', '.join([str(a) for a in args] + kwarg_strs)
+        call_str = '{}({})'.format(function.func_name, arg_str)
+
+        # Perform the call itself, logging before, after, and anything thrown.
+        try:
+            if log_calls:
+                print '{}: Calling {}'.format(label, call_str)
+            res = function(*args, **kwargs)
+            if log_results:
+                print '{}: {} -> {}'.format(label, call_str, res)
+            return res
+        except Exception as ex:
+            if log_results:
+                print '{}: {} raised {}'.format(label, call_str, type(ex))
+            raise ex
+
+    return wrapper
+
+
+def trace_object(obj, log_calls, log_results, label=''):
+    for name, member in inspect.getmembers(obj):
+        if inspect.ismethod(member):
+            # Skip meta-functions, decorate everything else
+            if not member.func_name.startswith('__'):
+                setattr(obj, name, trace_function(member, log_calls,
+                                                  log_results, label))
+    return obj

test/lit.site.cfg.in

@@ -18,6 +18,7 @@ config.target_triple            = "@LIBCXX_TARGET_TRIPLE@"
 config.sysroot                  = "@LIBCXX_SYSROOT@"
 config.gcc_toolchain            = "@LIBCXX_GCC_TOOLCHAIN@"
 config.target_info              = "@LIBCXX_TARGET_INFO@"
+config.executor                 = "@LIBCXX_EXECUTOR@"
 
 # Let the main config do the real work.
 lit_config.load_config(config, "@LIBCXX_SOURCE_DIR@/test/lit.cfg")