ThreadSanitizer (TSAN) and AddressSanitizer (ASAN) bots

BUG=None
TEST=Tested on local master with the live slaves.

Review URL: https://webrtc-codereview.appspot.com/571011

git-svn-id: http://webrtc.googlecode.com/svn/trunk@2258 4adac7df-926f-26a2-2b94-8c16560cd09d

tools/.gitignore

@@ -38,14 +38,16 @@ twistd.pid
 /continuous_build/build_internal/masters/master.webrtc/android
 /continuous_build/build_internal/masters/master.webrtc/android-ndk
 /continuous_build/build_internal/masters/master.webrtc/chromeos
+/continuous_build/build_internal/masters/master.webrtc/linux-asan
 /continuous_build/build_internal/masters/master.webrtc/linux-clang
 /continuous_build/build_internal/masters/master.webrtc/linux-large-tests
+/continuous_build/build_internal/masters/master.webrtc/linux-memcheck
 /continuous_build/build_internal/masters/master.webrtc/linux-slave-1
 /continuous_build/build_internal/masters/master.webrtc/linux-slave-2
 /continuous_build/build_internal/masters/master.webrtc/linux-slave-4
 /continuous_build/build_internal/masters/master.webrtc/linux-slave-5
 /continuous_build/build_internal/masters/master.webrtc/linux-slave-gcc-4.6
-/continuous_build/build_internal/masters/master.webrtc/linux-valgrind
+/continuous_build/build_internal/masters/master.webrtc/linux-tsan
 /continuous_build/build_internal/masters/master.webrtc/LinuxChrome
 /continuous_build/build_internal/masters/master.webrtc/LinuxChromeBloat
 /continuous_build/build_internal/masters/master.webrtc/mac-large-tests/

tools/continuous_build/build_internal/masters/master.webrtc/master.cfg

@@ -47,7 +47,9 @@ webrtc_scheduler = Scheduler(name='all', branch='trunk',
                                            'Linux64DBG',
                                            'Linux64Release',
                                            'LinuxClang',
-                                           'LinuxValgrind',
+                                           'LinuxMemcheck',
+                                           'LinuxTsan',
+                                           'LinuxAsan',
                                            'Linux64DBG-GCC4.6',
                                            'LinuxLargeTests',
                                            'MacLargeTests',
@@ -114,10 +116,22 @@ PHYSICAL_MACHINE_TESTS = {
     'voe_auto_test':                   (True, True, True),
 }
 
-VALGRIND_DISABLED_TESTS = [
+MEMCHECK_DISABLED_TESTS = [
     'audio_coding_module_test', # Issue 270
     'test_fec',                 # Too slow for Valgrind memcheck
 ]
+TSAN_DISABLED_TESTS = [
+    'audio_coding_module_test',   # Issue 283
+    'audioproc_unittest',         # Issue 299
+    'system_wrappers_unittests',  # Issue 300
+    'udp_transport_unittests',    # Issue 536
+    'video_processing_unittests', # Issue 303
+    'test_fec',                   # Too slow for TSAN
+]
+ASAN_DISABLED_TESTS = [
+    'audio_coding_module_test', # Issue 281
+    'neteq_unittests',          # Issue 282
+]
 
 linux_normal_tests = utils.GetEnabledTests(NORMAL_TESTS, 'Linux')
 mac_normal_tests = utils.GetEnabledTests(NORMAL_TESTS, 'Mac')
@@ -144,6 +158,10 @@ VALGRIND_DEPS_LIST = [
   ('trunk/third_party/valgrind',
     'http://src.chromium.org/svn/trunk/deps/third_party/valgrind/binaries'),
 ]
+ASAN_DEPS_LIST = [
+  ('trunk/third_party/asan',
+   'http://src.chromium.org/chrome/trunk/deps/third_party/asan'),
+]
 
 # Linux
 linux_factory_64_dbg = utils.WebRTCLinuxFactory(
@@ -188,14 +206,33 @@ linux_clang = utils.WebRTCLinuxFactory(
 linux_clang.EnableBuild(clang=True)
 linux_clang.EnableTests(linux_normal_tests)
 
-linux_valgrind = utils.WebRTCLinuxFactory(
+linux_memcheck_factory = utils.WebRTCLinuxFactory(
-    utils.BuildStatusOracle('linux_valgrind'), run_with_memcheck=True,
+    utils.BuildStatusOracle('linux_memcheck_factory'), run_with_memcheck=True,
     custom_deps_list=VALGRIND_DEPS_LIST)
-linux_valgrind.EnableBuild(release=True, compile_for_memory_tooling=True)
+linux_memcheck_factory.EnableBuild(release=True,
-# Filter out disabled Valgrind tests:
+                                   compile_for_memory_tooling=True)
-valgrind_tests = filter(lambda test: test not in VALGRIND_DISABLED_TESTS,
+# Filter out disabled memcheck tests:
+memcheck_tests = filter(lambda test: test not in MEMCHECK_DISABLED_TESTS,
                         linux_normal_tests)
-linux_valgrind.EnableTests(valgrind_tests)
+linux_memcheck_factory.EnableTests(memcheck_tests)
+
+linux_tsan_factory = utils.WebRTCLinuxFactory(
+    utils.BuildStatusOracle('linux_tsan_factory'), run_with_tsan=True,
+    custom_deps_list=VALGRIND_DEPS_LIST)
+linux_tsan_factory.EnableBuild(release=True, compile_for_memory_tooling=True)
+# Filter out disabled tsan tests:
+tsan_tests = filter(lambda test: test not in TSAN_DISABLED_TESTS,
+                    linux_normal_tests)
+linux_tsan_factory.EnableTests(tsan_tests)
+
+linux_asan_factory = utils.WebRTCLinuxFactory(
+    utils.BuildStatusOracle('linux_asan_factory'), run_with_asan=True,
+    custom_deps_list=ASAN_DEPS_LIST)
+linux_asan_factory.EnableBuild(release=True, clang=True)
+# Filter out disabled asan tests:
+asan_tests = filter(lambda test: test not in ASAN_DISABLED_TESTS,
+                    linux_normal_tests)
+linux_asan_factory.EnableTests(asan_tests)
 
 android_factory = utils.WebRTCAndroidFactory(
     utils.BuildStatusOracle('android_factory'))
@@ -316,11 +353,23 @@ linux_builder_clang = {
       'builddir': 'linux-clang',
       'factory': linux_clang,
       }
-linux_builder_valgrind = {
+linux_builder_memcheck = {
-      'name': 'LinuxValgrind',
+      'name': 'LinuxMemcheck',
       'slavename': 'webrtc-cb-linux-slave-6',
-      'builddir': 'linux-valgrind',
+      'builddir': 'linux-memcheck',
-      'factory': linux_valgrind,
+      'factory': linux_memcheck_factory,
+      }
+linux_builder_tsan = {
+      'name': 'LinuxTsan',
+      'slavename': 'webrtc-cb-linux-slave-15',
+      'builddir': 'linux-tsan',
+      'factory': linux_tsan_factory,
+      }
+linux_builder_asan = {
+      'name': 'LinuxAsan',
+      'slavename': 'webrtc-cb-linux-slave-16',
+      'builddir': 'linux-asan',
+      'factory': linux_asan_factory,
       }
 android_builder_1 = {
       'name': 'Android',
@@ -350,7 +399,9 @@ c['builders'] = [
       linux_builder_64_release,
       linux_builder_64_debug,
       linux_builder_clang,
-      linux_builder_valgrind,
+      linux_builder_memcheck,
+      linux_builder_tsan,
+      linux_builder_asan,
       linux_builder_gcc_4_6,
       win_builder_large_tests,
       mac_builder_large_tests,

tools/continuous_build/build_internal/masters/master.webrtc/slaves.cfg

@@ -55,7 +55,7 @@ def linux():
       'os': 'linux',
       'version': 'lucid',
       'bits': '64',
-      'builder': 'LinuxValgrind',
+      'builder': 'LinuxMemcheck',
       'hostname': 'webrtc-cb-linux-slave-6',
     },
     {
@@ -90,6 +90,22 @@ def linux():
       'builder': 'AndroidNDK',
       'hostname': 'webrtc-cb-linux-slave-10',
     },
+    {
+      'master': 'WebRTC',
+      'os': 'linux',
+      'version': 'lucid',
+      'bits': '64',
+      'builder': 'LinuxTsan',
+      'hostname': 'webrtc-cb-linux-slave-15',
+    },
+    {
+      'master': 'WebRTC',
+      'os': 'linux',
+      'version': 'lucid',
+      'bits': '64',
+      'builder': 'LinuxAsan',
+      'hostname': 'webrtc-cb-linux-slave-16',
+    },
     {
       'master': 'WebRTC',
       'os': 'linux',

tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py

@@ -33,6 +33,8 @@ WEBRTC_TRUNK_DIR = 'build/trunk'
 WEBRTC_BUILD_DIR = 'build'
 
 MEMCHECK_CMD = ['tools/valgrind-webrtc/webrtc_tests.sh', '-t']
+TSAN_CMD = ['tools/valgrind-webrtc/webrtc_tests.sh', '--tool', 'tsan', '-t']
+ASAN_CMD = ['tools/valgrind-webrtc/webrtc_tests.sh', '--tool', 'asan',  '-t']
 
 DEFAULT_COVERAGE_DIR = '/var/www/coverage'
 DEFAULT_MASTER_WORK_DIR = '.'
@@ -42,8 +44,8 @@ GCLIENT_RETRIES = 3
 # but converted to a list since we set defines instead of using an environment
 # variable.
 #
-# On valgrind bots, override the optimizer settings so we don't inline too
+# On memcheck and tsan bots, override the optimizer settings so we don't inline
-# much and make the stacks harder to figure out. Use the same settings
+# too much and make the stacks harder to figure out. Use the same settings
 # on all buildbot masters to make it easier to move bots.
 MEMORY_TOOLS_GYP_DEFINES = [
     # GCC flags
@@ -131,7 +133,7 @@ class WebRTCFactory(factory.BuildFactory):
 
   def AddCommonStep(self, cmd, descriptor='', workdir=WEBRTC_TRUNK_DIR,
                     halt_build_on_failure=True, warn_on_failure=False,
-                    timeout=1200, use_pty=True):
+                    timeout=1200, use_pty=True, env={}):
     """Adds a step which will run as a shell command on the slave.
 
     NOTE: you are recommended to use this method to add new shell commands
@@ -163,6 +165,8 @@ class WebRTCFactory(factory.BuildFactory):
         stderr output with red in the web interface. Some shell
         commands seem to fail when Pseudo-terminal is enabled on
         Linux.
+      env: dict of string->string that describes the environment the command
+        shall be excuted with on the build slave.
     """
     flunk_on_failure = not warn_on_failure
 
@@ -183,7 +187,8 @@ class WebRTCFactory(factory.BuildFactory):
         haltOnFailure=halt_build_on_failure,
         name='_'.join(descriptor),
         timeout=timeout,
-        usePTY=use_pty))
+        usePTY=use_pty,
+        env=env))
 
   def AddSmartCleanStep(self):
     """Adds a smart clean step.
@@ -519,13 +524,16 @@ class WebRTCLinuxFactory(WebRTCFactory):
   """
 
   def __init__(self, build_status_oracle, is_try_slave=False,
-               run_with_memcheck=False, custom_deps_list=None):
+               run_with_memcheck=False, run_with_tsan=False,
+               run_with_asan=False, custom_deps_list=None):
     WebRTCFactory.__init__(self, build_status_oracle=build_status_oracle,
                            is_try_slave=is_try_slave,
                            custom_deps_list=custom_deps_list)
     self.build_enabled = False
     self.coverage_enabled = False
     self.run_with_memcheck = run_with_memcheck
+    self.run_with_tsan = run_with_tsan
+    self.run_with_asan = run_with_asan
     self.compile_for_memory_tooling = False
 
   def EnableCoverage(self, coverage_url, coverage_dir=DEFAULT_COVERAGE_DIR):
@@ -555,15 +563,23 @@ class WebRTCLinuxFactory(WebRTCFactory):
     self.release = release
 
     self.AddSmartCleanStep()
+    self.AddGclientSyncStep()
 
     # Valgrind bots need special GYP defines to enable memory profiling
-    # friendly compilation. They already has a custom .gclient configuration
+    # friendly compilation.
-    # file created so they don't need one being generated like the other bots.
     if self.compile_for_memory_tooling:
       for gyp_define in MEMORY_TOOLS_GYP_DEFINES:
         self.gyp_params.append('-D' + gyp_define)
-    self.AddGclientSyncStep()
 
+    if self.run_with_asan:
+      # ASAN requires Clang compilation and we enforce Release mode since
+      # that's what Chromium recommends.
+      assert clang
+      assert release
+      self.gyp_params.append('-Dasan=1')
+      self.gyp_params.append('-Dlinux_use_tcmalloc=0')
+      self.gyp_params.append('-Drelease_extra_cflags="-g -O1 '
+                             '-fno-inline-functions -fno-inline"')
     if chrome_os:
       self.gyp_params.append('-Dchromeos=1')
 
@@ -591,6 +607,10 @@ class WebRTCLinuxFactory(WebRTCFactory):
       cmd = ['out/%s/%s' % (test_folder, test)]
     if self.run_with_memcheck:
       cmd = MEMCHECK_CMD + cmd
+    if self.run_with_tsan:
+      cmd = TSAN_CMD + cmd
+    if self.run_with_asan:
+      cmd = ASAN_CMD + cmd
     self.AddCommonStep(cmd, descriptor=descriptor, halt_build_on_failure=False)
 
   def AddXvfbTestRunStep(self, test_name, test_binary, test_arguments=''):
@@ -604,7 +624,18 @@ class WebRTCLinuxFactory(WebRTCFactory):
     cmd = ['make', target, '-j100']
     if make_extra:
       cmd.append(make_extra)
-    self.AddCommonStep(cmd=cmd, descriptor=descriptor)
+
+    env = {}
+    if self.run_with_asan:
+      # Override the Clang compiler with the prebuilt ASAN executables.
+      # It seems like CC and CXX must contain full paths and since there's no
+      # way to evaluate subcommands on the slaves, this hardcoding is the only
+      # way for now.
+      asan_bin = ('/b/build/slave/linux-asan/build/trunk/third_party/asan/'
+                  'asan_clang_Linux/bin')
+      env = {'CC': '%s/clang' % asan_bin,
+             'CXX': '%s/clang++ ' % asan_bin}
+    self.AddCommonStep(cmd=cmd, descriptor=descriptor, env=env)
 
   def AddStepsToEstablishCoverageBaseline(self):
     self.AddCommonStep(['lcov', '--directory', '.', '--capture', '-b',