From 5eeaa385731e8e88444d41993f5ed59fe148b3cb Mon Sep 17 00:00:00 2001 From: "kjellander@webrtc.org" Date: Sat, 4 Feb 2012 09:41:26 +0000 Subject: [PATCH] Improved readability of tests in master.cfg and enabling some tests Focused responsibility of supported tests in master.cfg instead of being in utils.py (hard to overview and maintain). Enabled the following empty tests on all platforms: - audio_conference_mixer_unittests - cng_unittests - g711_unittests - g722_unittests - pcm16b_unittests - media_file_unittests - udp_transport_unittests - webrtc_utility_unittests Removed "headless tests" concept since everything is now compiled in the make all step (no need for compile only, no execution tests). Removed audio_device_test_func test since not a proper test (dev tool) that was configured as headless. BUG= TEST=Ran local master and successfully built and executed all tests with Mac build slave. Review URL: http://webrtc-codereview.appspot.com/384002 git-svn-id: http://webrtc.googlecode.com/svn/trunk@1603 4adac7df-926f-26a2-2b94-8c16560cd09d --- tools/continuous_build/master.cfg | 158 ++++++-------- .../continuous_build/webrtc_buildbot/utils.py | 193 ++++-------------- .../webrtc_buildbot/utils_test.py | 40 ++++ 3 files changed, 138 insertions(+), 253 deletions(-) create mode 100644 tools/continuous_build/webrtc_buildbot/utils_test.py diff --git a/tools/continuous_build/master.cfg b/tools/continuous_build/master.cfg index c8bfb8771..751937a70 100755 --- a/tools/continuous_build/master.cfg +++ b/tools/continuous_build/master.cfg @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. +# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. # # Use of this source code is governed by a BSD-style license # that can be found in the LICENSE file in the root of the source @@ -106,142 +106,102 @@ from buildbot.process import factory from buildbot.steps import shell from webrtc_buildbot import utils -DEFAULT_LINUX_TESTS = ["audio_coding_module_test", - "audio_coding_unittests", - "audio_conference_mixer_unittests", - "audio_device_test_api", - "audio_device_test_func", - "audioproc_unittest", - "cng_unittests", - "g711_unittests", - "g722_unittests", - "libyuv_unittests", - "media_file_unittests", - "neteq_unittests", - "pcm16b_unittests" - "resampler_unittests", - "rtp_rtcp_unittests", - "signal_processing_unittests", - "system_wrappers_unittests", - "test_bwe", - # TODO(andrew): enable test_fec (and below) when its - # running time is reduced. - #"test_fec", - "udp_transport_unittests", - "vad_unittests", - "video_coding_unittests", - "video_engine_core_unittests", - "video_processing_unittests", - "voice_engine_unittests", - "vp8_unittests", - "webrtc_utility_unittests"] -DEFAULT_MACOS_TESTS = ["audio_coding_module_test", - "audio_coding_unittests", - "audio_conference_mixer_unittests", - "audio_device_test_api", - "audio_device_test_func", - "audioproc_unittest", - "cng_unittests", - "g711_unittests", - "g722_unittests", - "libyuv_unittests", - "media_file_unittests", - "neteq_unittests", - "pcm16b_unittests" - "resampler_unittests", - "rtp_rtcp_unittests", - "signal_processing_unittests", - "system_wrappers_unittests", - "test_bwe", - #"test_fec", - "udp_transport_unittests", - "vad_unittests", - "video_coding_unittests", - "video_engine_core_unittests", - "video_processing_unittests", - "voice_engine_unittests", - "vp8_unittests", - "webrtc_utility_unittests"] +# Tests to run on Virtual machine bots and our Mac hardware. +# Defines the supported tests followed by a tuple defining if the tests are +# enabled on Linux, Mac and/or Windows (in that order; defined in utils.py). +NORMAL_TESTS = { + # Test name Linux Mac Windows + "audio_coding_module_test": (True, True, False), + "audio_coding_unittests": (True, True, True), + "audio_conference_mixer_unittests":(True, True, False), + "audio_device_test_api": (True, True, False), + "audioproc_unittest": (True, True, True), + "cng_unittests": (True, True, True), + "g711_unittests": (True, True, True), + "g722_unittests": (True, True, True), + "libyuv_unittests": (True, True, True), + "media_file_unittests": (True, True, False), + "neteq_unittests": (True, True, True), + "pcm16b_unittests": (True, True, True), + "resampler_unittests": (True, True, True), + "rtp_rtcp_unittests": (True, True, False), + "signal_processing_unittests": (True, True, False), + "system_wrappers_unittests": (True, True, True), + "test_bwe": (True, True, False), + "test_fec": (False, False, False), + "udp_transport_unittests": (True, True, False), + "vad_unittests": (True, True, True), + "video_coding_unittests": (True, True, False), + "video_engine_core_unittests": (True, True, True), + "video_processing_unittests": (True, True, False), # Issue #247 + "voice_engine_unittests": (True, True, True), + "vp8_unittests": (True, True, True), + "webrtc_utility_unittests": (True, True, False), +} -# TODO(andrew): most tests should now run properly on Windows. -DEFAULT_WIN_TESTS = ["audio_coding_unittests", - "audioproc_unittest", - "libyuv_unittests", - "neteq_unittests", - "resampler_unittests", - "system_wrappers_unittests", - #"test_fec" - "vad_unittests", - "video_engine_core_unittests", - # TODO(andrew): enable when this is fixed: - # http://code.google.com/p/webrtc/issues/detail?id=247 - #"video_processing_unittests", - "voice_engine_unittests", - "vp8_unittests"] +PHYSICAL_MACHINE_TESTS = { + # Test name Linux Mac Windows + "vie_auto_test": (True, False, False), + "voe_auto_test": (True, False, False), +} -HEADLESS_LINUX = ["audio_device_test_api", - "audio_device_test_func"] -HEADLESS_MACOS = ["audio_device_test_api", - "audio_device_test_func"] -HEADLESS_WIN = ["audio_device_test_api", - "audio_device_test_func"] +linux_normal_tests = utils.GetEnabledTests(NORMAL_TESTS, "Linux") +mac_normal_tests = utils.GetEnabledTests(NORMAL_TESTS, "Mac") +windows_normal_tests = utils.GetEnabledTests(NORMAL_TESTS, "Windows") + +linux_physical_machine_tests = utils.GetEnabledTests(PHYSICAL_MACHINE_TESTS, + "Linux") +mac_physical_machine_tests = utils.GetEnabledTests(PHYSICAL_MACHINE_TESTS, + "Mac") +windows_physical_machine_tests = utils.GetEnabledTests(PHYSICAL_MACHINE_TESTS, + "Windows") ############# Linux Builders ####################################### linux_factory_64_dbg = utils.WebRTCLinuxFactory(enable_coverage=True) linux_factory_64_dbg.EnableBuild() -linux_factory_64_dbg.EnableHeadLess(HEADLESS_LINUX) -linux_factory_64_dbg.EnableTests(DEFAULT_LINUX_TESTS) +linux_factory_64_dbg.EnableTests(linux_normal_tests) linux_factory_64_dbg_no_coverage = utils.WebRTCLinuxFactory() linux_factory_64_dbg_no_coverage.EnableBuild() -linux_factory_64_dbg_no_coverage.EnableHeadLess(HEADLESS_LINUX) -linux_factory_64_dbg_no_coverage.EnableTests(DEFAULT_LINUX_TESTS) +linux_factory_64_dbg_no_coverage.EnableTests(linux_normal_tests) linux_factory_32_release = utils.WebRTCLinuxFactory(enable_coverage=True) linux_factory_32_release.EnableBuild(release=True, build32=True) -linux_factory_32_release.EnableHeadLess(HEADLESS_LINUX) -linux_factory_32_release.EnableTests(DEFAULT_LINUX_TESTS) +linux_factory_32_release.EnableTests(linux_normal_tests) linux_factory_64_release = utils.WebRTCLinuxFactory(enable_coverage=True) linux_factory_64_release.EnableBuild(release=True) -linux_factory_64_release.EnableHeadLess(HEADLESS_LINUX) -linux_factory_64_release.EnableTests(DEFAULT_LINUX_TESTS) +linux_factory_64_release.EnableTests(linux_normal_tests) linux_factory_32_dbg = utils.WebRTCLinuxFactory(enable_coverage=True) linux_factory_32_dbg.EnableBuild(build32=True) -linux_factory_32_dbg.EnableHeadLess(HEADLESS_LINUX) -linux_factory_32_dbg.EnableTests(DEFAULT_LINUX_TESTS) +linux_factory_32_dbg.EnableTests(linux_normal_tests) linux_factory_video = utils.WebRTCLinuxFactory() linux_factory_video.EnableBuild() -linux_factory_video.EnableTests(["vie_auto_test", "voe_auto_test"]) +linux_factory_video.EnableTests(linux_physical_machine_tests) chromeos_factory = utils.WebRTCLinuxFactory() chromeos_factory.EnableBuild(chrome_os=True) -chromeos_factory.EnableHeadLess(HEADLESS_LINUX) -chromeos_factory.EnableTests(DEFAULT_LINUX_TESTS) +chromeos_factory.EnableTests(linux_normal_tests) linux_clang = utils.WebRTCLinuxFactory() linux_clang.EnableBuild(clang=True) -linux_clang.EnableHeadLess(HEADLESS_LINUX) -linux_clang.EnableTests(DEFAULT_LINUX_TESTS) +linux_clang.EnableTests(linux_normal_tests) ############# Mac Builders ####################################### mac_factory = utils.WebRTCMacFactory() mac_factory.EnableBuild(build_type="both") -mac_factory.EnableHeadLess(HEADLESS_MACOS) -mac_factory.EnableTests(DEFAULT_MACOS_TESTS) +mac_factory.EnableTests(mac_normal_tests) ############# Windows Builders ####################################### win_factory_32_Debug = utils.WebRTCWinFactory() win_factory_32_Debug.EnableBuild(configuration="Debug") -win_factory_32_Debug.EnableHeadLess(HEADLESS_WIN) -win_factory_32_Debug.EnableTests(DEFAULT_WIN_TESTS) +win_factory_32_Debug.EnableTests(windows_normal_tests) win_factory_32_Release = utils.WebRTCWinFactory() win_factory_32_Release.EnableBuild(configuration="Release") -win_factory_32_Release.EnableHeadLess(HEADLESS_WIN) -win_factory_32_Release.EnableTests(DEFAULT_WIN_TESTS) +win_factory_32_Release.EnableTests(windows_normal_tests) android_factory = utils.WebRTCAndroidFactory() android_factory.EnableBuild(product="toro") diff --git a/tools/continuous_build/webrtc_buildbot/utils.py b/tools/continuous_build/webrtc_buildbot/utils.py index c9c4c4f83..6c2051bb5 100755 --- a/tools/continuous_build/webrtc_buildbot/utils.py +++ b/tools/continuous_build/webrtc_buildbot/utils.py @@ -17,6 +17,9 @@ from buildbot.process.properties import WithProperties import os import sys +# Defines the order of the booleans of the supported platforms in the test +# dictionaries in master.cfg. +SUPPORTED_PLATFORMS = ("Linux", "Mac", "Windows") SVN_LOCATION = "http://webrtc.googlecode.com/svn/trunk" @@ -29,7 +32,6 @@ class WebRTCFactory(factory.BuildFactory): self.properties = properties.Properties() self.enable_build = False self.force_sync = False - self.headless_tests = [] self.enable_coverage = enable_coverage self.gyp_params = [] self.account = account @@ -86,18 +88,19 @@ class WebRTCFactory(factory.BuildFactory): name="gyp_%s" % descriptor)) def EnableTest(self, test): - """Enable Test to be run. [must be overridden] + """Enable Test to be run. May be overridden. + + Override to handle special cases for specific platforms. test: test to be run. """ - pass + self.AddCommonTestRunStep(test) def EnableTests(self, tests): """Enable Test to be run. tests: list of test to be run. """ - print "Headless tests:%s" % self.headless_tests if self.enable_coverage: self.EnableBaseCoverage() for test in tests: @@ -105,13 +108,6 @@ class WebRTCFactory(factory.BuildFactory): if self.enable_coverage: self.EnableCoverage() - def EnableHeadLess(self, tests): - """Add headless (build only) tests. - - tests: list of headless test. - """ - self.headless_tests += tests - def EnableBaseCoverage(self): """Enable base coverage data [must be overridden].""" pass @@ -178,7 +174,7 @@ class WebRTCAndroidFactory(WebRTCFactory): description=["cleanup", "running..."], haltOnFailure=False, warnOnFailure=True, flunkOnFailure =False, descriptionDone=["cleanup", "done..."], name="cleanup")) - cmd = "svn checkout http://webrtc.googlecode.com/svn/trunk/ external/webrtc" + cmd = "svn checkout %s external/webrtc" % SVN_LOCATION self.addStep(shell.Compile(command=(cmd), workdir="build/trunk", description=["svn", "running..."], haltOnFailure=False, descriptionDone=["svn", "done..."], name="svn")) @@ -263,8 +259,6 @@ class WebRTCLinuxFactory(WebRTCFactory): def AddCommonTestRunStep(self, test, descriptor="", cmd=None, workdir="build/trunk"): - if test in self.headless_tests: - return test_folder = "Release" if self.release else "Debug" test_descriptor = [test, descriptor] if cmd is None: @@ -340,58 +334,6 @@ class WebRTCLinuxFactory(WebRTCFactory): descriptor="fixed_point") self.AddCommonMakeStep(test, descriptor="make_fixed_point") self.AddCommonTestRunStep(test, descriptor="fixed_point") - elif test == "signal_processing_unittests": - self.AddCommonTestRunStep(test) - elif test == "resampler_unittests": - self.AddCommonTestRunStep(test) - elif test == "vad_unittests": - self.AddCommonTestRunStep(test) - elif test == "rtp_rtcp_unittests": - self.AddCommonTestRunStep(test) - elif test == "video_coding_unittests": - self.AddCommonTestRunStep(test) - elif test == "test_bwe": - self.AddCommonTestRunStep(test) - elif test == "audio_device_test_api": - self.AddCommonTestRunStep(test) - elif test == "audio_device_test_func": - self.AddCommonTestRunStep(test) - elif test == "audio_coding_module_test": - self.AddCommonTestRunStep(test) - elif test == "video_processing_unittests": - self.AddCommonTestRunStep(test) - elif test == "test_fec": - self.AddCommonTestRunStep(test) - elif test == "system_wrappers_unittests": - self.AddCommonTestRunStep(test) - elif test == "cng_unittests": - self.AddCommonTestRunStep(test) - elif test == "g711_unittests": - self.AddCommonTestRunStep(test) - elif test == "g722_unittests": - self.AddCommonTestRunStep(test) - elif test == "pcm16b_unittests": - self.AddCommonTestRunStep(test) - elif test == "audio_conference_mixer_unittests": - self.AddCommonTestRunStep(test) - elif test == "media_file_unittests": - self.AddCommonTestRunStep(test) - elif test == "udp_transport_unittests": - self.AddCommonTestRunStep(test) - elif test == "webrtc_utility_unittests": - self.AddCommonTestRunStep(test) - elif test == "neteq_unittests": - self.AddCommonTestRunStep(test) - elif test == "vp8_unittests": - self.AddCommonTestRunStep(test) - elif test == "libyuv_unittests": - self.AddCommonTestRunStep(test) - elif test == "voice_engine_unittests": - self.AddCommonTestRunStep(test) - elif test == "video_engine_core_unittests": - self.AddCommonTestRunStep(test) - elif test == "audio_coding_unittests": - self.AddCommonTestRunStep(test) elif test == "vie_auto_test": self.addStep(shell.Compile(command=('xvfb-run --server-args="-screen 0 ' '800x600x24 -extension Composite" out/Debug/vie_auto_test --automated ' @@ -405,7 +347,7 @@ class WebRTCLinuxFactory(WebRTCFactory): workdir="build/trunk", description=[test, "running..."], descriptionDone=[test, "done..."], name="%s" % test)) else: - print "[Linux]: No supported tests are found for [%s]" % test + self.AddCommonTestRunStep(test) ################################################################################ @@ -443,8 +385,6 @@ class WebRTCMacFactory(WebRTCFactory): def AddCommonTestRunStep(self, test, descriptor="", cmd=None, workdir="build/trunk"): - if test in self.headless_tests: - return test_folder = "Release" if self.release else "Debug" test_descriptor = [test, descriptor] if cmd is None: @@ -472,60 +412,6 @@ class WebRTCMacFactory(WebRTCFactory): self.AddCommonStep(cmd, descriptor=make_descriptor+["(xcode)"], workdir="build/trunk") - def EnableTest(self, test): - """Enable Test to be run. - - test: test to be run. - """ - if test == "audioproc_unittest": - self.AddCommonTestRunStep(test) - elif test == "signal_processing_unittests": - self.AddCommonTestRunStep(test) - elif test == "resampler_unittests": - self.AddCommonTestRunStep(test) - elif test == "vad_unittests": - self.AddCommonTestRunStep(test) - elif test == "rtp_rtcp_unittests": - self.AddCommonTestRunStep(test) - elif test == "video_coding_unittests": - self.AddCommonTestRunStep(test) - elif test == "test_bwe": - self.AddCommonTestRunStep(test) - elif test == "audio_device_test_api": - self.AddCommonTestRunStep(test) - elif test == "audio_device_test_func": - self.AddCommonTestRunStep(test) - elif test == "audio_coding_module_test": - self.AddCommonTestRunStep(test) - elif test == "video_processing_unittests": - self.AddCommonTestRunStep(test) - elif test == "test_fec": - self.AddCommonTestRunStep(test) - elif test == "system_wrappers_unittests": - self.AddCommonTestRunStep(test) - elif test == "audio_conference_mixer_unittests": - self.AddCommonTestRunStep(test) - elif test == "media_file_unittests": - self.AddCommonTestRunStep(test) - elif test == "udp_transport_unittests": - self.AddCommonTestRunStep(test) - elif test == "webrtc_utility_unittests": - self.AddCommonTestRunStep(test) - elif test == "neteq_unittests": - self.AddCommonTestRunStep(test) - elif test == "vp8_unittests": - self.AddCommonTestRunStep(test) - elif test == "libyuv_unittests": - self.AddCommonTestRunStep(test) - elif test == "voice_engine_unittests": - self.AddCommonTestRunStep(test) - elif test == "video_engine_core_unittests": - self.AddCommonTestRunStep(test) - elif test == "audio_coding_unittests": - self.AddCommonTestRunStep(test) - else: - print "[Mac]: No supported tests are found for [%s]" % test - ################################################################################ class WebRTCWinFactory(WebRTCFactory): """A Build Factory affected by properties.""" @@ -580,8 +466,6 @@ class WebRTCWinFactory(WebRTCFactory): def AddCommonTestRunStep(self, test, descriptor="", cmd=None, workdir="build/trunk"): - if test in self.headless_tests: - return test_descriptor = [test, descriptor] if cmd is None: if self.configuration == "Debug" or self.configuration == "both": @@ -593,35 +477,36 @@ class WebRTCWinFactory(WebRTCFactory): self.AddCommonStep(cmd, descriptor=test_descriptor+["Release"], workdir=workdir) +################################################################################ +# Utility functions - def EnableTest(self, test): - """Enable Test to be run. - - test: test to be run. - """ - if test == "audioproc_unittest": - self.AddCommonTestRunStep(test) - elif test == "resampler_unittests": - self.AddCommonTestRunStep(test) - elif test == "vad_unittests": - self.AddCommonTestRunStep(test) - elif test == "system_wrappers_unittests": - self.AddCommonTestRunStep(test) - elif test == "neteq_unittests": - self.AddCommonTestRunStep(test) - elif test == "vp8_unittests": - self.AddCommonTestRunStep(test) - elif test == "libyuv_unittests": - self.AddCommonTestRunStep(test) - elif test == "voice_engine_unittests": - self.AddCommonTestRunStep(test) - elif test == "video_engine_core_unittests": - self.AddCommonTestRunStep(test) - elif test == "video_processing_unittests": - self.AddCommonTestRunStep(test) - elif test == "audio_coding_unittests": - self.AddCommonTestRunStep(test) - else: - print "[Win]: No supported tests are found for [%s]" % test +class UnsupportedPlatformError(Exception): + pass +def GetEnabledTests(test_dict, platform): + """Returns a list of enabled test names for the provided platform. + + Args: + test_dict: Dictionary mapping test names to tuples representing if the + test shall be enabled on each platform. Each tuple contains one boolean + for each platform. The platforms are in the order specified by + SUPPORTED_PLATFORMS. + platform: The platform we're looking to get the tests for. + + Returns: + A list of test names, sorted alphabetically. + + Raises: + UnsupportedPlatformError: if the platform supplied is not supported. + """ + if platform not in SUPPORTED_PLATFORMS: + raise UnsupportedPlatformError("*** UNSUPPORTED PLATFORM (%s)!!! ***" % + platform) + result = [] + platform_index = SUPPORTED_PLATFORMS.index(platform) + for test_name, enabled_platforms in test_dict.iteritems(): + if enabled_platforms[platform_index]: + result.append(test_name) + result.sort() + return result \ No newline at end of file diff --git a/tools/continuous_build/webrtc_buildbot/utils_test.py b/tools/continuous_build/webrtc_buildbot/utils_test.py new file mode 100644 index 000000000..559f4ec7e --- /dev/null +++ b/tools/continuous_build/webrtc_buildbot/utils_test.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +__author__ = 'kjellander@webrtc.org (Henrik Kjellander)' + +import unittest + +from webrtc_buildbot import utils + +class Test(unittest.TestCase): + + def testGetEnabledTests(self): + tests = { + # Test name Linux Mac Windows + "test_1": (True, True, False), + "test_2": (True, False, False), + } + result = utils.GetEnabledTests(tests, "Linux") + self.assertEqual(2, len(result)) + self.assertEqual('test_1', result[0]) + self.assertEqual('test_2', result[1]) + + result = utils.GetEnabledTests(tests, "Mac") + self.assertEqual(1, len(result)) + self.assertEqual('test_1', result[0]) + + result = utils.GetEnabledTests(tests, "Windows") + self.assertEqual(0, len(result)) + + self.assertRaises(utils.UnsupportedPlatformError, + utils.GetEnabledTests, tests, "BeOS") + +if __name__ == "__main__": + unittest.main()