generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improved readability of tests in master.cfg and enabling some tests

Browse Source 
Focused responsibility of supported tests in master.cfg instead of being in utils.py (hard to overview and maintain).

Enabled the following empty tests on all platforms:
- audio_conference_mixer_unittests
- cng_unittests
- g711_unittests
- g722_unittests
- pcm16b_unittests
- media_file_unittests
- udp_transport_unittests
- webrtc_utility_unittests

Removed "headless tests" concept since everything is now compiled in the make all step (no need for compile only, no execution tests).
Removed audio_device_test_func test since not a proper test (dev tool) that was configured as headless.

BUG=
TEST=Ran local master and successfully built and executed all tests with Mac build slave.

Review URL: http://webrtc-codereview.appspot.com/384002

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1603 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
kjellander@webrtc.org 2012-02-04 09:41:26 +00:00
parent 4942832928
commit 5eeaa38573
3 changed files with 138 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
tools/continuous_build/master.cfg
View File

@ -1,5 +1,5 @@
#!/usr/bin/env python
# Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
@ -106,142 +106,102 @@ from buildbot.process import factory
from buildbot.steps import shell
from webrtc_buildbot import utils
DEFAULT_LINUX_TESTS = ["audio_coding_module_test",
"audio_coding_unittests",
"audio_conference_mixer_unittests",
"audio_device_test_api",
"audio_device_test_func",
"audioproc_unittest",
"cng_unittests",
"g711_unittests",
"g722_unittests",
"libyuv_unittests",
"media_file_unittests",
"neteq_unittests",
"pcm16b_unittests"
"resampler_unittests",
"rtp_rtcp_unittests",
"signal_processing_unittests",
"system_wrappers_unittests",
"test_bwe",
# TODO(andrew): enable test_fec (and below) when its
# running time is reduced.
#"test_fec",
"udp_transport_unittests",
"vad_unittests",
"video_coding_unittests",
"video_engine_core_unittests",
"video_processing_unittests",
"voice_engine_unittests",
"vp8_unittests",
"webrtc_utility_unittests"]
DEFAULT_MACOS_TESTS = ["audio_coding_module_test",
"audio_coding_unittests",
"audio_conference_mixer_unittests",
"audio_device_test_api",
"audio_device_test_func",
"audioproc_unittest",
"cng_unittests",
"g711_unittests",
"g722_unittests",
"libyuv_unittests",
"media_file_unittests",
"neteq_unittests",
"pcm16b_unittests"
"resampler_unittests",
"rtp_rtcp_unittests",
"signal_processing_unittests",
"system_wrappers_unittests",
"test_bwe",
#"test_fec",
"udp_transport_unittests",
"vad_unittests",
"video_coding_unittests",
"video_engine_core_unittests",
"video_processing_unittests",
"voice_engine_unittests",
"vp8_unittests",
"webrtc_utility_unittests"]
# Tests to run on Virtual machine bots and our Mac hardware.
# Defines the supported tests followed by a tuple defining if the tests are
# enabled on Linux, Mac and/or Windows (in that order; defined in utils.py).
NORMAL_TESTS = {
# Test name Linux Mac Windows
"audio_coding_module_test": (True, True, False),
"audio_coding_unittests": (True, True, True),
"audio_conference_mixer_unittests":(True, True, False),
"audio_device_test_api": (True, True, False),
"audioproc_unittest": (True, True, True),
"cng_unittests": (True, True, True),
"g711_unittests": (True, True, True),
"g722_unittests": (True, True, True),
"libyuv_unittests": (True, True, True),
"media_file_unittests": (True, True, False),
"neteq_unittests": (True, True, True),
"pcm16b_unittests": (True, True, True),
"resampler_unittests": (True, True, True),
"rtp_rtcp_unittests": (True, True, False),
"signal_processing_unittests": (True, True, False),
"system_wrappers_unittests": (True, True, True),
"test_bwe": (True, True, False),
"test_fec": (False, False, False),
"udp_transport_unittests": (True, True, False),
"vad_unittests": (True, True, True),
"video_coding_unittests": (True, True, False),
"video_engine_core_unittests": (True, True, True),
"video_processing_unittests": (True, True, False), # Issue #247
"voice_engine_unittests": (True, True, True),
"vp8_unittests": (True, True, True),
"webrtc_utility_unittests": (True, True, False),
}
# TODO(andrew): most tests should now run properly on Windows.
DEFAULT_WIN_TESTS = ["audio_coding_unittests",
"audioproc_unittest",
"libyuv_unittests",
"neteq_unittests",
"resampler_unittests",
"system_wrappers_unittests",
#"test_fec"
"vad_unittests",
"video_engine_core_unittests",
# TODO(andrew): enable when this is fixed:
# http://code.google.com/p/webrtc/issues/detail?id=247
#"video_processing_unittests",
"voice_engine_unittests",
"vp8_unittests"]
PHYSICAL_MACHINE_TESTS = {
# Test name Linux Mac Windows
"vie_auto_test": (True, False, False),
"voe_auto_test": (True, False, False),
}
HEADLESS_LINUX = ["audio_device_test_api",
"audio_device_test_func"]
HEADLESS_MACOS = ["audio_device_test_api",
"audio_device_test_func"]
HEADLESS_WIN = ["audio_device_test_api",
"audio_device_test_func"]
linux_normal_tests = utils.GetEnabledTests(NORMAL_TESTS, "Linux")
mac_normal_tests = utils.GetEnabledTests(NORMAL_TESTS, "Mac")
windows_normal_tests = utils.GetEnabledTests(NORMAL_TESTS, "Windows")
linux_physical_machine_tests = utils.GetEnabledTests(PHYSICAL_MACHINE_TESTS,
"Linux")
mac_physical_machine_tests = utils.GetEnabledTests(PHYSICAL_MACHINE_TESTS,
"Mac")
windows_physical_machine_tests = utils.GetEnabledTests(PHYSICAL_MACHINE_TESTS,
"Windows")
############# Linux Builders #######################################
linux_factory_64_dbg = utils.WebRTCLinuxFactory(enable_coverage=True)
linux_factory_64_dbg.EnableBuild()
linux_factory_64_dbg.EnableHeadLess(HEADLESS_LINUX)
linux_factory_64_dbg.EnableTests(DEFAULT_LINUX_TESTS)
linux_factory_64_dbg.EnableTests(linux_normal_tests)
linux_factory_64_dbg_no_coverage = utils.WebRTCLinuxFactory()
linux_factory_64_dbg_no_coverage.EnableBuild()
linux_factory_64_dbg_no_coverage.EnableHeadLess(HEADLESS_LINUX)
linux_factory_64_dbg_no_coverage.EnableTests(DEFAULT_LINUX_TESTS)
linux_factory_64_dbg_no_coverage.EnableTests(linux_normal_tests)
linux_factory_32_release = utils.WebRTCLinuxFactory(enable_coverage=True)
linux_factory_32_release.EnableBuild(release=True, build32=True)
linux_factory_32_release.EnableHeadLess(HEADLESS_LINUX)
linux_factory_32_release.EnableTests(DEFAULT_LINUX_TESTS)
linux_factory_32_release.EnableTests(linux_normal_tests)
linux_factory_64_release = utils.WebRTCLinuxFactory(enable_coverage=True)
linux_factory_64_release.EnableBuild(release=True)
linux_factory_64_release.EnableHeadLess(HEADLESS_LINUX)
linux_factory_64_release.EnableTests(DEFAULT_LINUX_TESTS)
linux_factory_64_release.EnableTests(linux_normal_tests)
linux_factory_32_dbg = utils.WebRTCLinuxFactory(enable_coverage=True)
linux_factory_32_dbg.EnableBuild(build32=True)
linux_factory_32_dbg.EnableHeadLess(HEADLESS_LINUX)
linux_factory_32_dbg.EnableTests(DEFAULT_LINUX_TESTS)
linux_factory_32_dbg.EnableTests(linux_normal_tests)
linux_factory_video = utils.WebRTCLinuxFactory()
linux_factory_video.EnableBuild()
linux_factory_video.EnableTests(["vie_auto_test", "voe_auto_test"])
linux_factory_video.EnableTests(linux_physical_machine_tests)
chromeos_factory = utils.WebRTCLinuxFactory()
chromeos_factory.EnableBuild(chrome_os=True)
chromeos_factory.EnableHeadLess(HEADLESS_LINUX)
chromeos_factory.EnableTests(DEFAULT_LINUX_TESTS)
chromeos_factory.EnableTests(linux_normal_tests)
linux_clang = utils.WebRTCLinuxFactory()
linux_clang.EnableBuild(clang=True)
linux_clang.EnableHeadLess(HEADLESS_LINUX)
linux_clang.EnableTests(DEFAULT_LINUX_TESTS)
linux_clang.EnableTests(linux_normal_tests)
############# Mac Builders #######################################
mac_factory = utils.WebRTCMacFactory()
mac_factory.EnableBuild(build_type="both")
mac_factory.EnableHeadLess(HEADLESS_MACOS)
mac_factory.EnableTests(DEFAULT_MACOS_TESTS)
mac_factory.EnableTests(mac_normal_tests)
############# Windows Builders #######################################
win_factory_32_Debug = utils.WebRTCWinFactory()
win_factory_32_Debug.EnableBuild(configuration="Debug")
win_factory_32_Debug.EnableHeadLess(HEADLESS_WIN)
win_factory_32_Debug.EnableTests(DEFAULT_WIN_TESTS)
win_factory_32_Debug.EnableTests(windows_normal_tests)
win_factory_32_Release = utils.WebRTCWinFactory()
win_factory_32_Release.EnableBuild(configuration="Release")
win_factory_32_Release.EnableHeadLess(HEADLESS_WIN)
win_factory_32_Release.EnableTests(DEFAULT_WIN_TESTS)
win_factory_32_Release.EnableTests(windows_normal_tests)
android_factory = utils.WebRTCAndroidFactory()
android_factory.EnableBuild(product="toro")

193
tools/continuous_build/webrtc_buildbot/utils.py
View File

@ -17,6 +17,9 @@ from buildbot.process.properties import WithProperties
import os
import sys
# Defines the order of the booleans of the supported platforms in the test
# dictionaries in master.cfg.
SUPPORTED_PLATFORMS = ("Linux", "Mac", "Windows")
SVN_LOCATION = "http://webrtc.googlecode.com/svn/trunk"
@ -29,7 +32,6 @@ class WebRTCFactory(factory.BuildFactory):
self.properties = properties.Properties()
self.enable_build = False
self.force_sync = False
self.headless_tests = []
self.enable_coverage = enable_coverage
self.gyp_params = []
self.account = account
@ -86,18 +88,19 @@ class WebRTCFactory(factory.BuildFactory):
name="gyp_%s" % descriptor))
def EnableTest(self, test):
"""Enable Test to be run. [must be overridden]
"""Enable Test to be run. May be overridden.
Override to handle special cases for specific platforms.
test: test to be run.
"""
pass
self.AddCommonTestRunStep(test)
def EnableTests(self, tests):
"""Enable Test to be run.
tests: list of test to be run.
"""
print "Headless tests:%s" % self.headless_tests
if self.enable_coverage:
self.EnableBaseCoverage()
for test in tests:
@ -105,13 +108,6 @@ class WebRTCFactory(factory.BuildFactory):
if self.enable_coverage:
self.EnableCoverage()
def EnableHeadLess(self, tests):
"""Add headless (build only) tests.
tests: list of headless test.
"""
self.headless_tests += tests
def EnableBaseCoverage(self):
"""Enable base coverage data [must be overridden]."""
pass
@ -178,7 +174,7 @@ class WebRTCAndroidFactory(WebRTCFactory):
description=["cleanup", "running..."], haltOnFailure=False,
warnOnFailure=True, flunkOnFailure =False,
descriptionDone=["cleanup", "done..."], name="cleanup"))
cmd = "svn checkout http://webrtc.googlecode.com/svn/trunk/ external/webrtc"
cmd = "svn checkout %s external/webrtc" % SVN_LOCATION
self.addStep(shell.Compile(command=(cmd),
workdir="build/trunk", description=["svn", "running..."],
haltOnFailure=False, descriptionDone=["svn", "done..."], name="svn"))
@ -263,8 +259,6 @@ class WebRTCLinuxFactory(WebRTCFactory):
def AddCommonTestRunStep(self, test, descriptor="", cmd=None,
workdir="build/trunk"):
if test in self.headless_tests:
return
test_folder = "Release" if self.release else "Debug"
test_descriptor = [test, descriptor]
if cmd is None:
@ -340,58 +334,6 @@ class WebRTCLinuxFactory(WebRTCFactory):
descriptor="fixed_point")
self.AddCommonMakeStep(test, descriptor="make_fixed_point")
self.AddCommonTestRunStep(test, descriptor="fixed_point")
elif test == "signal_processing_unittests":
self.AddCommonTestRunStep(test)
elif test == "resampler_unittests":
self.AddCommonTestRunStep(test)
elif test == "vad_unittests":
self.AddCommonTestRunStep(test)
elif test == "rtp_rtcp_unittests":
self.AddCommonTestRunStep(test)
elif test == "video_coding_unittests":
self.AddCommonTestRunStep(test)
elif test == "test_bwe":
self.AddCommonTestRunStep(test)
elif test == "audio_device_test_api":
self.AddCommonTestRunStep(test)
elif test == "audio_device_test_func":
self.AddCommonTestRunStep(test)
elif test == "audio_coding_module_test":
self.AddCommonTestRunStep(test)
elif test == "video_processing_unittests":
self.AddCommonTestRunStep(test)
elif test == "test_fec":
self.AddCommonTestRunStep(test)
elif test == "system_wrappers_unittests":
self.AddCommonTestRunStep(test)
elif test == "cng_unittests":
self.AddCommonTestRunStep(test)
elif test == "g711_unittests":
self.AddCommonTestRunStep(test)
elif test == "g722_unittests":
self.AddCommonTestRunStep(test)
elif test == "pcm16b_unittests":
self.AddCommonTestRunStep(test)
elif test == "audio_conference_mixer_unittests":
self.AddCommonTestRunStep(test)
elif test == "media_file_unittests":
self.AddCommonTestRunStep(test)
elif test == "udp_transport_unittests":
self.AddCommonTestRunStep(test)
elif test == "webrtc_utility_unittests":
self.AddCommonTestRunStep(test)
elif test == "neteq_unittests":
self.AddCommonTestRunStep(test)
elif test == "vp8_unittests":
self.AddCommonTestRunStep(test)
elif test == "libyuv_unittests":
self.AddCommonTestRunStep(test)
elif test == "voice_engine_unittests":
self.AddCommonTestRunStep(test)
elif test == "video_engine_core_unittests":
self.AddCommonTestRunStep(test)
elif test == "audio_coding_unittests":
self.AddCommonTestRunStep(test)
elif test == "vie_auto_test":
self.addStep(shell.Compile(command=('xvfb-run --server-args="-screen 0 '
'800x600x24 -extension Composite" out/Debug/vie_auto_test --automated '
@ -405,7 +347,7 @@ class WebRTCLinuxFactory(WebRTCFactory):
workdir="build/trunk", description=[test, "running..."],
descriptionDone=[test, "done..."], name="%s" % test))
else:
print "[Linux]: No supported tests are found for [%s]" % test
self.AddCommonTestRunStep(test)
################################################################################
@ -443,8 +385,6 @@ class WebRTCMacFactory(WebRTCFactory):
def AddCommonTestRunStep(self, test, descriptor="", cmd=None,
workdir="build/trunk"):
if test in self.headless_tests:
return
test_folder = "Release" if self.release else "Debug"
test_descriptor = [test, descriptor]
if cmd is None:
@ -472,60 +412,6 @@ class WebRTCMacFactory(WebRTCFactory):
self.AddCommonStep(cmd, descriptor=make_descriptor+["(xcode)"],
workdir="build/trunk")
def EnableTest(self, test):
"""Enable Test to be run.
test: test to be run.
"""
if test == "audioproc_unittest":
self.AddCommonTestRunStep(test)
elif test == "signal_processing_unittests":
self.AddCommonTestRunStep(test)
elif test == "resampler_unittests":
self.AddCommonTestRunStep(test)
elif test == "vad_unittests":
self.AddCommonTestRunStep(test)
elif test == "rtp_rtcp_unittests":
self.AddCommonTestRunStep(test)
elif test == "video_coding_unittests":
self.AddCommonTestRunStep(test)
elif test == "test_bwe":
self.AddCommonTestRunStep(test)
elif test == "audio_device_test_api":
self.AddCommonTestRunStep(test)
elif test == "audio_device_test_func":
self.AddCommonTestRunStep(test)
elif test == "audio_coding_module_test":
self.AddCommonTestRunStep(test)
elif test == "video_processing_unittests":
self.AddCommonTestRunStep(test)
elif test == "test_fec":
self.AddCommonTestRunStep(test)
elif test == "system_wrappers_unittests":
self.AddCommonTestRunStep(test)
elif test == "audio_conference_mixer_unittests":
self.AddCommonTestRunStep(test)
elif test == "media_file_unittests":
self.AddCommonTestRunStep(test)
elif test == "udp_transport_unittests":
self.AddCommonTestRunStep(test)
elif test == "webrtc_utility_unittests":
self.AddCommonTestRunStep(test)
elif test == "neteq_unittests":
self.AddCommonTestRunStep(test)
elif test == "vp8_unittests":
self.AddCommonTestRunStep(test)
elif test == "libyuv_unittests":
self.AddCommonTestRunStep(test)
elif test == "voice_engine_unittests":
self.AddCommonTestRunStep(test)
elif test == "video_engine_core_unittests":
self.AddCommonTestRunStep(test)
elif test == "audio_coding_unittests":
self.AddCommonTestRunStep(test)
else:
print "[Mac]: No supported tests are found for [%s]" % test
################################################################################
class WebRTCWinFactory(WebRTCFactory):
"""A Build Factory affected by properties."""
@ -580,8 +466,6 @@ class WebRTCWinFactory(WebRTCFactory):
def AddCommonTestRunStep(self, test, descriptor="", cmd=None,
workdir="build/trunk"):
if test in self.headless_tests:
return
test_descriptor = [test, descriptor]
if cmd is None:
if self.configuration == "Debug" or self.configuration == "both":
@ -593,35 +477,36 @@ class WebRTCWinFactory(WebRTCFactory):
self.AddCommonStep(cmd, descriptor=test_descriptor+["Release"],
workdir=workdir)
################################################################################
# Utility functions
def EnableTest(self, test):
"""Enable Test to be run.
test: test to be run.
"""
if test == "audioproc_unittest":
self.AddCommonTestRunStep(test)
elif test == "resampler_unittests":
self.AddCommonTestRunStep(test)
elif test == "vad_unittests":
self.AddCommonTestRunStep(test)
elif test == "system_wrappers_unittests":
self.AddCommonTestRunStep(test)
elif test == "neteq_unittests":
self.AddCommonTestRunStep(test)
elif test == "vp8_unittests":
self.AddCommonTestRunStep(test)
elif test == "libyuv_unittests":
self.AddCommonTestRunStep(test)
elif test == "voice_engine_unittests":
self.AddCommonTestRunStep(test)
elif test == "video_engine_core_unittests":
self.AddCommonTestRunStep(test)
elif test == "video_processing_unittests":
self.AddCommonTestRunStep(test)
elif test == "audio_coding_unittests":
self.AddCommonTestRunStep(test)
else:
print "[Win]: No supported tests are found for [%s]" % test
class UnsupportedPlatformError(Exception):
pass
def GetEnabledTests(test_dict, platform):
"""Returns a list of enabled test names for the provided platform.
Args:
test_dict: Dictionary mapping test names to tuples representing if the
test shall be enabled on each platform. Each tuple contains one boolean
for each platform. The platforms are in the order specified by
SUPPORTED_PLATFORMS.
platform: The platform we're looking to get the tests for.
Returns:
A list of test names, sorted alphabetically.
Raises:
UnsupportedPlatformError: if the platform supplied is not supported.
"""
if platform not in SUPPORTED_PLATFORMS:
raise UnsupportedPlatformError("*** UNSUPPORTED PLATFORM (%s)!!! ***" %
platform)
result = []
platform_index = SUPPORTED_PLATFORMS.index(platform)
for test_name, enabled_platforms in test_dict.iteritems():
if enabled_platforms[platform_index]:
result.append(test_name)
result.sort()
return result

40
tools/continuous_build/webrtc_buildbot/utils_test.py Normal file
View File

@ -0,0 +1,40 @@
#!/usr/bin/env python
# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
__author__ = 'kjellander@webrtc.org (Henrik Kjellander)'
import unittest
from webrtc_buildbot import utils
class Test(unittest.TestCase):
def testGetEnabledTests(self):
tests = {
# Test name Linux Mac Windows
"test_1": (True, True, False),
"test_2": (True, False, False),
}
result = utils.GetEnabledTests(tests, "Linux")
self.assertEqual(2, len(result))
self.assertEqual('test_1', result[0])
self.assertEqual('test_2', result[1])
result = utils.GetEnabledTests(tests, "Mac")
self.assertEqual(1, len(result))
self.assertEqual('test_1', result[0])
result = utils.GetEnabledTests(tests, "Windows")
self.assertEqual(0, len(result))
self.assertRaises(utils.UnsupportedPlatformError,
utils.GetEnabledTests, tests, "BeOS")
if __name__ == "__main__":
unittest.main()