Made fuzz tests and chrome bots FYI: e.g, they will warn instead of making the whole build red if they fail.

BUG=
TEST=

Review URL: https://webrtc-codereview.appspot.com/589005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@2286 4adac7df-926f-26a2-2b94-8c16560cd09d

tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py

@@ -132,8 +132,8 @@ class WebRTCFactory(factory.BuildFactory):
       self.EnableTest(test)
 
   def AddCommonStep(self, cmd, descriptor='', workdir=WEBRTC_TRUNK_DIR,
-                    halt_build_on_failure=True, warn_on_failure=False,
+                    halt_build_on_failure=True, timeout=1200, use_pty=True,
-                    timeout=1200, use_pty=True, env={}):
+                    env={}):
     """Adds a step which will run as a shell command on the slave.
 
     NOTE: you are recommended to use this method to add new shell commands
@@ -154,10 +154,7 @@ class WebRTCFactory(factory.BuildFactory):
         way depending on platform, which means you can't use the default
         value if the step will run on a Windows machine.
       halt_build_on_failure: Stops the build dead in its tracks if this step
-        fails. Use for critical steps. This option does not make sense with
+        fails. Use for critical steps.
-        warn_on_failure.
-      warn_on_failure: If true, this step isn't that important and will not
-        cause a failed build on failure.
       timeout: The timeout for the command, in seconds.
       use_pty: If Pseudo-terminal shall be enabled for the command. This is
         needed if stdout and stderr output shall be collected
@@ -168,28 +165,42 @@ class WebRTCFactory(factory.BuildFactory):
       env: dict of string->string that describes the environment the command
         shall be excuted with on the build slave.
     """
-    flunk_on_failure = not warn_on_failure
+    description, description_done = self._FormatDescriptor(descriptor)
-
-    if type(descriptor) is str:
-      descriptor = [descriptor]
-
-    # Add spaces to wrap long test names to make waterfall output more compact.
-    wrapped_text = self._WrapLongLines(descriptor)
 
     self.addStep(MonitoredShellCommand(
         build_status_oracle=self.build_status_oracle,
         command=cmd,
         workdir=workdir,
-        description=wrapped_text + ['running...'],
+        description=description,
-        descriptionDone=wrapped_text,
+        descriptionDone=description_done,
-        warnOnFailure=warn_on_failure,
+        flunkOnFailure=True,
-        flunkOnFailure=flunk_on_failure,
         haltOnFailure=halt_build_on_failure,
-        name='_'.join(descriptor),
+        name='_'.join(description_done),
         timeout=timeout,
         usePTY=use_pty,
         env=env))
 
+  def AddCommonFyiStep(self, cmd, descriptor='', workdir=WEBRTC_TRUNK_DIR):
+    """Adds a command which is merely FYI.
+
+    This command will only produce a warning on failure and will not be
+    considered a failure by the build status oracle.
+
+    The parameters here have the same semantics as their counterparts in
+    AddCommonStep.
+    """
+    description, description_done = self._FormatDescriptor(descriptor)
+
+    self.addStep(ShellCommand(
+        command=cmd,
+        workdir=workdir,
+        description=description,
+        descriptionDone=description_done,
+        name='_'.join(description_done),
+        flunkOnFailure=False,
+        haltOnFailure=False,
+        warnOnFailure=True))
+
   def AddSmartCleanStep(self):
     """Adds a smart clean step.
 
@@ -330,6 +341,23 @@ class WebRTCFactory(factory.BuildFactory):
       result.append(line)
     return result
 
+  def _FormatDescriptor(self, descriptor):
+    """Formats the descriptor.
+
+    Args:
+      descriptor: A string or list describing the build step.
+
+    Returns:
+      A tuple containing the formatted descriptor as well as a suitable
+      descriptor to use when the step is done.
+    """
+    if type(descriptor) is str:
+      descriptor = [descriptor]
+
+    # Add spaces to wrap long test names to make waterfall output more compact.
+    wrapped_text = self._WrapLongLines(descriptor)
+    return (wrapped_text + ['running'], wrapped_text)
+
 
 class BuildStatusOracle:
   """Keeps track of a particular build's state.
@@ -613,11 +641,6 @@ class WebRTCLinuxFactory(WebRTCFactory):
       cmd = ASAN_CMD + cmd
     self.AddCommonStep(cmd, descriptor=descriptor, halt_build_on_failure=False)
 
-  def AddXvfbTestRunStep(self, test_name, test_binary, test_arguments=''):
-    """ Adds a test to be run inside a XVFB window manager."""
-    cmd = MakeCommandToRunTestInXvfb('%s %s' % (test_binary, test_arguments))
-    self.AddCommonTestRunStep(test=test_name, cmd=cmd)
-
   def AddCommonMakeStep(self, target, extra_text=None, make_extra=None):
     descriptor = ['make ' + target, extra_text] if extra_text else ['make ' +
                                                                     target]
@@ -638,24 +661,18 @@ class WebRTCLinuxFactory(WebRTCFactory):
     self.AddCommonStep(cmd=cmd, descriptor=descriptor, env=env)
 
   def AddStepsToEstablishCoverageBaseline(self):
-    self.AddCommonStep(['lcov', '--directory', '.', '--capture', '-b',
+    self.AddCommonFyiStep(['lcov', '--directory', '.', '--capture', '-b',
-                        '.', '--initial',
+                           '.', '--initial',
-                        '--output-file', 'webrtc_base.info'],
+                           '--output-file', 'webrtc_base.info'],
-                       warn_on_failure=True,
+                           descriptor='LCOV (Baseline Capture)')
-                       halt_build_on_failure=False,
+    self.AddCommonFyiStep(['lcov', '--extract', 'webrtc_base.info', '*/src/*',
-                       descriptor='LCOV (Baseline Capture)')
+                           '--output', 'filtered.info'],
-    self.AddCommonStep(['lcov', '--extract', 'webrtc_base.info', '*/src/*',
+                           descriptor='LCOV (Baseline Extract)')
-                        '--output', 'filtered.info'],
+    self.AddCommonFyiStep(['lcov', '--remove', 'filtered.info',
-                       warn_on_failure=True,
+                           '*/usr/include/*', '/third*', '/testing/*',
-                       halt_build_on_failure=False,
+                           '*/test/*', '*_unittest.*', '*/mock/*', '--output',
-                       descriptor='LCOV (Baseline Extract)')
+                           'webrtc_base_filtered_final.info'],
-    self.AddCommonStep(['lcov', '--remove', 'filtered.info', '*/usr/include/*',
+                           descriptor='LCOV (Baseline Filter)')
-                        '/third*', '/testing/*', '*/test/*', '*_unittest.*',
-                        '*/mock/*', '--output',
-                        'webrtc_base_filtered_final.info'],
-                       warn_on_failure=True,
-                       halt_build_on_failure=False,
-                       descriptor='LCOV (Baseline Filter)')
 
   def AddStepsToComputeCoverage(self):
     """Enable coverage data."""
@@ -664,34 +681,24 @@ class WebRTCLinuxFactory(WebRTCFactory):
     # in lcov which tends to hang when capturing on libjpgturbo.
     clean_script = PosixPathJoin('tools', 'continuous_build', 'build_internal',
                                  'scripts', 'clean_third_party_gcda.sh')
-    self.AddCommonStep([clean_script],
+    self.AddCommonFyiStep([clean_script],
-                       warn_on_failure=True,
+                          descriptor='LCOV (Delete 3rd party)')
-                       halt_build_on_failure=False,
+    self.AddCommonFyiStep(['lcov', '--directory', '.', '--capture', '-b',
-                       descriptor='LCOV (Delete 3rd party)')
+                           '.', '--output-file', 'webrtc.info'],
-    self.AddCommonStep(['lcov', '--directory', '.', '--capture', '-b',
+                          descriptor='LCOV (Capture)')
-                        '.', '--output-file', 'webrtc.info'],
+    self.AddCommonFyiStep(['lcov', '--extract', 'webrtc.info', '*/src/*',
-                       warn_on_failure=True,
+                           '--output', 'test.info'],
-                       halt_build_on_failure=False,
+                          descriptor='LCOV (Extract)')
-                       descriptor='LCOV (Capture)')
+    self.AddCommonFyiStep(['lcov', '--remove', 'test.info', '*/usr/include/*',
-    self.AddCommonStep(['lcov', '--extract', 'webrtc.info', '*/src/*',
+                           '/third*', '/testing/*', '*/test/*', '*_unittest.*',
-                        '--output', 'test.info'],
+                           '*/mock/*', '--output',
-                       warn_on_failure=True,
+                           'final.info'],
-                       halt_build_on_failure=False,
+                          descriptor='LCOV (Filter)')
-                       descriptor='LCOV (Extract)')
+    self.AddCommonFyiStep(['lcov', '-a',
-    self.AddCommonStep(['lcov', '--remove', 'test.info', '*/usr/include/*',
+                           'webrtc_base_filtered_final.info', '-a',
-                        '/third*', '/testing/*', '*/test/*', '*_unittest.*',
+                           'final.info', '-o', 'final.info'],
-                        '*/mock/*', '--output',
+                          descriptor='LCOV (Merge)')
-                        'final.info'],
-                       warn_on_failure=True,
-                       halt_build_on_failure=False,
-                       descriptor='LCOV (Filter)')
-    self.AddCommonStep(['lcov', '-a', 'webrtc_base_filtered_final.info', '-a',
-                        'final.info', '-o', 'final.info'],
-                       warn_on_failure=True,
-                       halt_build_on_failure=False,
-                       descriptor='LCOV (Merge)')
 
-    # This step isn't monitored but it's fine since it's not critical.
     self.addStep(
         GenerateCodeCoverage(build_status_oracle=self.build_status_oracle,
                              coverage_url=self.coverage_url,
@@ -725,16 +732,13 @@ class WebRTCLinuxFactory(WebRTCFactory):
       self.AddCommonMakeStep(test, extra_text='(fixed point)')
       self.AddCommonTestRunStep(test, extra_text='(fixed point)')
     elif test == 'vie_auto_test':
-      # TODO(phoglund): Enable the full stack test once it is completed and
-      # nonflaky.
       binary = 'out/Debug/vie_auto_test'
       filter = '-ViEVideoVerificationTest.RunsFullStack*:ViERtpFuzzTest*'
-      args = (
+      cmd = [binary, '--automated', '--gtest_filter=%s' % filter,
-        '--automated --gtest_filter="%s" '
+             ('--capture_test_ensure_resolution_alignment'
-        '--capture_test_ensure_resolution_alignment_in_capture_device=false')
+              '_in_capture_device=false')]
-      args = args % filter
+      cmd = MakeCommandToRunTestInXvfb(cmd)
-      self.AddXvfbTestRunStep(test_name=test, test_binary=binary,
+      self.AddCommonTestRunStep(test=test, cmd=cmd)
-                              test_arguments=args)
 
       # Set up the fuzz tests as a separate step under memcheck.
       # If this test is run we require that we have compiled for memory tools.
@@ -742,21 +746,28 @@ class WebRTCLinuxFactory(WebRTCFactory):
       # when calling the webrtc_tests.sh script since we want those parameters
       # to not be caught by webrtc_tests.sh's options parser, but be passed on
       # to vie_auto_test. This is a part of webrtc_tests.sh's contract.
+      # This test is considered to be a FYI test so we will only warn here.
       assert self.compile_for_memory_tooling
-      fuzz_binary = (' '.join(MEMCHECK_CMD) + ' ' + binary +
+      fuzz_cmd = MEMCHECK_CMD + [binary, '++automated',
-                     ' ++automated ++gtest_filter=ViERtpFuzzTest*')
+                                 '++gtest_filter=ViERtpFuzzTest*']
-      self.AddXvfbTestRunStep(test_name=test + ' (fuzz tests)',
+      fuzz_cmd = MakeCommandToRunTestInXvfb(fuzz_cmd)
-                              test_binary=fuzz_binary)
+      self.AddCommonFyiStep(cmd=fuzz_cmd, descriptor=test + ' (fuzz tests)')
     elif test == 'video_render_module_test':
-      self.AddXvfbTestRunStep(test_name=test,
+      cmd = MakeCommandToRunTestInXvfb(['out/Debug/video_render_module_test'])
-                              test_binary='out/Debug/video_render_module_test')
-    elif test == 'voe_auto_test':
-      cmd = 'out/Debug/voe_auto_test --automated'
       self.AddCommonTestRunStep(test=test, cmd=cmd)
+    elif test == 'voe_auto_test':
+      binary = 'out/Debug/voe_auto_test'
+      cmd = [binary, '--automated', '--gtest_filter=-RtpFuzzTest.*']
+      self.AddCommonTestRunStep(test=test, cmd=cmd)
+
+      # Similarly to vie_auto_test, set up voe_auto_test fuzz tests.
+      assert self.compile_for_memory_tooling
+      cmd = MEMCHECK_CMD + [binary, ' ++automated',
+                            '++gtest_filter=RtpFuzzTest*']
+      self.AddCommonFyiStep(cmd=cmd, descriptor='voe_auto_test (fuzz tests)')
     else:
       self.AddCommonTestRunStep(test)
 
-
 class WebRTCMacFactory(WebRTCFactory):
   """Sets up the Mac build, both for make and xcode."""
 
@@ -867,10 +878,10 @@ class WebRTCWinFactory(WebRTCFactory):
     self.path_joiner = WindowsPathJoin
 
   def AddCommonStep(self, cmd, descriptor='', workdir=WEBRTC_TRUNK_DIR,
-                    halt_build_on_failure=True, warn_on_failure=False):
+                    halt_build_on_failure=True):
     workdir = workdir.replace('/', '\\')
     WebRTCFactory.AddCommonStep(self, cmd, descriptor, workdir,
-                                halt_build_on_failure, warn_on_failure)
+                                halt_build_on_failure)
 
   def EnableBuild(self, platform='Win32', configuration='Debug'):
     if platform not in self.allowed_platforms:
@@ -995,8 +1006,9 @@ def WindowsPathJoin(*args):
   return ntpath.normpath(ntpath.join(*args))
 
 def MakeCommandToRunTestInXvfb(cmd):
+  assert type(cmd) is list
   return ('xvfb-run --server-args="-screen 0 800x600x24 -extension Composite" '
-          '%s' % cmd)
+          '%s' % ' '.join(cmd))
 
 
 class UnsupportedConfigurationError(Exception):

tools/continuous_build/build_internal/scripts/webrtc_buildbot/webrtc_commands.py

@@ -14,8 +14,8 @@ __author__ = 'kjellander@webrtc.org (Henrik Kjellander)'
 This is based on chromium_commands.py and adds WebRTC-specific commands."""
 
 from buildbot.steps.shell import ShellCommand
-
 from master.factory import chromium_commands
+from master.log_parser import cl_command
 from webrtc_buildbot import utils
 
 DEFAULT_BLOAT_DIR = '/var/www/bloat'
@@ -96,6 +96,45 @@ class WebRTCCommands(chromium_commands.ChromiumCommands):
                                               workdir='build/src',
                                               timeout=7200))
 
+  def AddTestStep(self, command_class, test_name, test_command,
+                  test_description='', timeout=10*60, max_time=8*60*60,
+                  workdir=None, env=None, locks=None, halt_on_failure=False,
+                  do_step_if=True):
+    """This override is a hack to get the step to warn instead of failing."""
+    assert timeout <= max_time
+    do_step_if = do_step_if or self.TestStepFilter
+    self._factory.addStep(
+        command_class,
+        name=test_name,
+        timeout=timeout,
+        maxTime=max_time,
+        doStepIf=do_step_if,
+        workdir=workdir,
+        env=env,
+        description='running %s%s' % (test_name, test_description),
+        descriptionDone='%s%s' % (test_name, test_description),
+        haltOnFailure=halt_on_failure,
+        warnOnFailure=True,
+        flunkOnFailure=False,
+        command=test_command)
+
+
+  def AddCompileStep(self, solution, clobber=False, description='compiling',
+                     descriptionDone='compile', timeout=600, mode=None,
+                     options=None):
+    """This override is a hack to get the step to warn instead of failing."""
+    self._factory.addStep(cl_command.CLCommand,
+                          enable_warnings=0,
+                          timeout=timeout,
+                          description=description,
+                          descriptionDone=descriptionDone,
+                          warnOnFailure=True,
+                          flunkOnFailure=False,
+                          command=self.GetBuildCommand(clobber,
+                                                       solution,
+                                                       mode,
+                                                       options))
+
 
 class ShellCommandWithUrl(ShellCommand):
   """A regular shell command which posts a link when it's done."""