generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Implemented quick builds (e.g. only do full clean if the previous build failed).

Browse Source 
BUG=
TEST=

Review URL: https://webrtc-codereview.appspot.com/400009

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1752 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
phoglund@webrtc.org 2012-02-23 12:34:01 +00:00
parent 6eb0ca2e75
commit dc3179dfe7
3 changed files with 158 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tools/continuous_build/cleanup.py → tools/continuous_build/clean_old_files.py
View File

@ -14,7 +14,7 @@ import os
import sys
import time
# The path is considered whitelisted if any of these entries appear
# The path is considered whitelisted if any of these entries appear
# at some point in the path
WHITELIST = ["buildbot.tac", "master.cfg", "public_html", "changes.pck",
"webrtc_buildbot"]

49
tools/continuous_build/master.cfg
View File

@ -173,70 +173,79 @@ windows_physical_machine_tests = utils.GetEnabledTests(PHYSICAL_MACHINE_TESTS,
'Windows')
############# Linux Builders #######################################
linux_factory_64_dbg = utils.WebRTCLinuxFactory()
linux_factory_64_dbg = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_factory_64_dbg'))
linux_factory_64_dbg.EnableBuild()
linux_factory_64_dbg.EnableTests(linux_normal_tests)
linux_factory_64_dbg_no_coverage = utils.WebRTCLinuxFactory()
linux_factory_64_dbg_no_coverage.EnableBuild()
linux_factory_64_dbg_no_coverage.EnableTests(linux_normal_tests)
linux_factory_32_release = utils.WebRTCLinuxFactory()
linux_factory_32_release = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_factory_32_release'))
linux_factory_32_release.EnableBuild(release=True, build32=True)
linux_factory_32_release.EnableTests(linux_normal_tests)
linux_factory_64_release = utils.WebRTCLinuxFactory()
linux_factory_64_release = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_factory_64_release'))
linux_factory_64_release.EnableBuild(release=True)
linux_factory_64_release.EnableTests(linux_normal_tests)
linux_factory_32_dbg = utils.WebRTCLinuxFactory()
linux_factory_32_dbg = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_factory_32_dbg'))
linux_factory_32_dbg.EnableCoverage(
coverage_url='http://webrtc-cb-linux-slave-4.cbf.corp.google.com/coverage/')
linux_factory_32_dbg.EnableBuild(build32=True)
linux_factory_32_dbg.EnableTests(linux_normal_tests)
linux_factory_video = utils.WebRTCLinuxFactory()
linux_factory_video = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_factory_video'))
linux_factory_video.EnableCoverage(
coverage_url='http://webrtc-build-bot-se.lul/coverage/')
linux_factory_video.EnableBuild()
linux_factory_video.EnableTests(linux_physical_machine_tests)
chromeos_factory = utils.WebRTCLinuxFactory()
chromeos_factory = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('chromeos_factory'))
chromeos_factory.EnableBuild(chrome_os=True)
chromeos_factory.EnableTests(linux_normal_tests)
linux_clang = utils.WebRTCLinuxFactory()
linux_clang = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_clang'))
linux_clang.EnableBuild(clang=True)
linux_clang.EnableTests(linux_normal_tests)
linux_valgrind = utils.WebRTCLinuxFactory(valgrind_enabled=True)
linux_valgrind = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_valgrind'), valgrind_enabled=True)
linux_valgrind.EnableBuild(release=True)
# Filter out disabled Valgrind tests:
valgrind_tests = filter(lambda test: test not in VALGRIND_DISABLED_TESTS,
linux_normal_tests)
linux_valgrind.EnableTests(valgrind_tests)
android_factory = utils.WebRTCAndroidFactory(
utils.BuildStatusOracle('android_factory'))
android_factory.EnableBuild(product='toro')
############## Mac Builders #######################################
mac_factory_32_dbg = utils.WebRTCMacFactory()
mac_factory_32_dbg = utils.WebRTCMacFactory(
utils.BuildStatusOracle('mac_factory_32_dbg'))
mac_factory_32_dbg.EnableBuild(build_type='both')
mac_factory_32_dbg.EnableTests(mac_normal_tests)
mac_factory_32_release = utils.WebRTCMacFactory()
mac_factory_32_release = utils.WebRTCMacFactory(
utils.BuildStatusOracle('mac_factory_32_release'))
mac_factory_32_release.EnableBuild(build_type='both', release=True)
mac_factory_32_release.EnableTests(mac_normal_tests)
############# Windows Builders #######################################
win_factory_32_Debug = utils.WebRTCWinFactory()
win_factory_32_Debug = utils.WebRTCWinFactory(
utils.BuildStatusOracle('win_factory_32_debug'))
win_factory_32_Debug.EnableBuild(configuration='Debug')
win_factory_32_Debug.EnableTests(windows_normal_tests)
win_factory_32_Release = utils.WebRTCWinFactory()
win_factory_32_Release = utils.WebRTCWinFactory(
utils.BuildStatusOracle('mac_factory_32_release'))
win_factory_32_Release.EnableBuild(configuration='Release')
win_factory_32_Release.EnableTests(windows_normal_tests)
android_factory = utils.WebRTCAndroidFactory()
android_factory.EnableBuild(product='toro')
linux_builder_64_debug = {
'name': 'Linux64DBG',
'slavename': 'linux-slave-1',
@ -319,7 +328,7 @@ linux_builder_gcc_4_6 = {
'name': 'Linux64DBG-GCC4.6',
'slavename': 'linux-slave-gcc-4.6',
'builddir': 'linux-slave-gcc-4.6',
'factory': linux_factory_64_dbg_no_coverage,
'factory': linux_factory_64_dbg,
}
c['builders'] = [
win_builder_32_debug,

161
tools/continuous_build/webrtc_buildbot/utils.py
View File

@ -9,6 +9,7 @@
__author__ = 'ivinnichenko@webrtc.org (Illya Vinnichenko)'
import buildbot
import os
import sys
import urlparse
@ -26,6 +27,7 @@ SVN_LOCATION = 'http://webrtc.googlecode.com/svn/trunk'
VALGRIND_CMD = ['tools/valgrind-webrtc/webrtc_tests.sh', '-t', 'cmdline']
DEFAULT_COVERAGE_DIR = '/var/www/coverage/'
DEFAULT_MASTER_WORK_DIR = '.'
# Copied from trunk/tools/build/scripts/master/factory/chromium_factory.py
# but converted to a list since we set defines instead of using an environment
@ -66,9 +68,16 @@ class WebRTCFactory(factory.BuildFactory):
can be overridden to create customized build sequences.
"""
def __init__(self):
def __init__(self, build_status_oracle):
"""Creates the abstract factory.
Args:
build_status_oracle: An instance of BuildStatusOracle which is used to
keep track of our build state.
"""
factory.BuildFactory.__init__(self)
self.build_status_oracle = build_status_oracle
self.properties = properties.Properties()
self.build_enabled = False
self.force_sync = False
@ -101,9 +110,13 @@ class WebRTCFactory(factory.BuildFactory):
def AddCommonStep(self, cmd, descriptor='', workdir='build',
halt_build_on_failure=True, warn_on_failure=False):
"""Adds a common step which will run as a shell command on the slave.
"""Adds a step which will run as a shell command on the slave.
A common step can be anything except a test execution step.
NOTE: you are recommended to use this method to add new shell commands
instead of the base-class addStep method, since steps added here will
work with the smart-clean system (e.g. only do a full rebuild if the
previous build failed). Steps handled outside this method will not lead
to a full rebuild on the next build if they fail.
Args:
cmd: The command to run. This command follows the contract for
@ -124,13 +137,20 @@ class WebRTCFactory(factory.BuildFactory):
# Add spaces to wrap long test names to make waterfall output more compact.
wrapped_text = self._WrapLongLines(descriptor)
self.addStep(shell.ShellCommand(command=cmd, workdir=workdir,
description=wrapped_text + ['running...'],
descriptionDone=wrapped_text,
warnOnFailure=warn_on_failure,
flunkOnFailure=flunk_on_failure,
haltOnFailure=halt_build_on_failure,
name='_'.join(descriptor)))
self.addStep(MonitoredShellCommand(
build_status_oracle=self.build_status_oracle, command=cmd,
workdir=workdir, description=wrapped_text + ['running...'],
descriptionDone=wrapped_text, warnOnFailure=warn_on_failure,
flunkOnFailure=flunk_on_failure, haltOnFailure=halt_build_on_failure,
name='_'.join(descriptor)))
def AddSmartCleanStep(self):
"""Adds a smart clean step.
Smart clean only cleans the whole repository if the build status oracle
thinks the last build failed. Otherwise it cleans just the build output.
"""
self.addStep(SmartClean(self.build_status_oracle))
def AddCommonTestRunStep(self, test, descriptor='', cmd=None,
workdir='build/trunk'):
@ -139,6 +159,9 @@ class WebRTCFactory(factory.BuildFactory):
In general, failing tests should not halt the build and allow other tests
to execute. A failing test should fail, or 'flunk', the build though.
Implementations of this method must add new steps through AddCommonStep
and not by calling addStep.
Args:
test: The test binary name. The step will attempt to execute this
binary in the binary output folder, except if the cmd argument is
@ -174,6 +197,9 @@ class WebRTCFactory(factory.BuildFactory):
GYP will generate makefiles or its equivalent in a platform-specific
manner. A failed GYP step will halt the build.
Implementations of this method must add new steps through AddCommonStep
and not by calling addStep.
Args:
gyp_file: The root GYP file to use.
gyp_params: Custom GYP parameters (same semantics as the GYP_PARAMS
@ -215,6 +241,80 @@ class WebRTCFactory(factory.BuildFactory):
result.append(line)
return result
class BuildStatusOracle:
"""Keeps track of a particular build's state.
The oracle uses files in the default master work directory to keep track
of whether a build has failed. It only keeps track of the most recent build
until told to forget it.
"""
def __init__(self, builder_name):
"""Creates the oracle.
Args:
builder_name: The name of the associated builder. This name is used
in the filename on disk. This name should be unique.
"""
self.builder_name = builder_name
self.master_work_dir = DEFAULT_MASTER_WORK_DIR
def LastBuildFailed(self):
failure_file_path = self._GetFailureBuildPath()
return os.path.exists(failure_file_path)
def ForgetLastBuild(self):
if self.LastBuildFailed():
os.remove(self._GetFailureBuildPath())
def SetLastBuildAsFailed(self):
open(self._GetFailureBuildPath(), 'w').close()
def _GetFailureBuildPath(self):
return os.path.join(self.master_work_dir, self.builder_name + ".failed")
class MonitoredShellCommand(ShellCommand):
"""Wraps a shell command and notifies the oracle if the command fails."""
def __init__(self, build_status_oracle, **kwargs):
ShellCommand.__init__(self, **kwargs)
self.addFactoryArguments(build_status_oracle=build_status_oracle)
self.build_status_oracle = build_status_oracle
def finished(self, results):
if (results == buildbot.status.builder.FAILURE or
results == buildbot.status.builder.EXCEPTION):
self.build_status_oracle.SetLastBuildAsFailed()
ShellCommand.finished(self, results)
class SmartClean(ShellCommand):
"""Cleans the repository fully or partially depending on the build state."""
def __init__(self, build_status_oracle, **kwargs):
ShellCommand.__init__(self, **kwargs)
self.addFactoryArguments(build_status_oracle=build_status_oracle)
self.haltOnFailure = True
self.build_status_oracle = build_status_oracle
def start(self):
if self.build_status_oracle.LastBuildFailed():
self.build_status_oracle.ForgetLastBuild()
self.description = ['Nuke Repository', '(Previous Failed)']
self.setCommand(['rm', '-rf', 'trunk'])
else:
self.description = ['Clean']
self.setCommand('rm -rf trunk/out && '
'rm -rf trunk/xcodebuild &&'
'rm -rf trunk/build/Debug &&'
'rm -rf trunk/build/Release')
ShellCommand.start(self)
class GenerateCodeCoverage(ShellCommand):
"""This custom shell command generates coverage HTML using genhtml.
@ -265,8 +365,8 @@ class GenerateCodeCoverage(ShellCommand):
class WebRTCAndroidFactory(WebRTCFactory):
"""Sets up the Android build."""
def __init__(self):
WebRTCFactory.__init__(self)
def __init__(self, build_status_oracle):
WebRTCFactory.__init__(self, build_status_oracle)
def EnableBuild(self, product='toro'):
prefix = 'rm -rf out/target/product/%s/obj/' % product
@ -277,25 +377,21 @@ class WebRTCAndroidFactory(WebRTCFactory):
prefix + 'EXECUTABLES/webrtc_*'
]
cmd = ' ; '.join(cleanup_list)
self.addStep(shell.Compile(command=(cmd), workdir='build/trunk',
description=['cleanup', 'running...'],
descriptionDone=['cleanup'], name='cleanup'))
self.AddCommonStep(cmd, descriptor='cleanup', workdir='build/trunk')
cmd = 'svn checkout %s external/webrtc' % SVN_LOCATION
self.addStep(shell.Compile(command=(cmd),
workdir='build/trunk', description=['svn checkout', 'running...'],
descriptionDone=['svn checkout'], name='svn (checkout)'))
self.AddCommonStep(cmd, descriptor='svn (checkout)', workdir='build/trunk')
cmd = ('source build/envsetup.sh && lunch full_%s-eng '
'&& mmm external/webrtc showcommands' % product)
self.addStep(shell.Compile(command=(cmd),
workdir='build/trunk', description=['build', 'running...'],
descriptionDone=['build'], name='build'))
self.AddCommonStep(cmd, descriptor='build', workdir='build/trunk')
class WebRTCChromeFactory(WebRTCFactory):
"""Sets up the Chrome OS build."""
def __init__(self):
WebRTCFactory.__init__(self)
def __init__(self, build_status_oracle):
WebRTCFactory.__init__(self, build_status_oracle)
def EnableBuild(self):
self.AddCommonStep(['rm', '-rf', 'src'], descriptor='Cleanup')
@ -317,8 +413,8 @@ class WebRTCLinuxFactory(WebRTCFactory):
This factory is quite configurable and can run a variety of builds.
"""
def __init__(self, valgrind_enabled=False):
WebRTCFactory.__init__(self)
def __init__(self, build_status_oracle, valgrind_enabled=False):
WebRTCFactory.__init__(self, build_status_oracle)
self.coverage_enabled = False
self.valgrind_enabled = valgrind_enabled
@ -349,8 +445,7 @@ class WebRTCLinuxFactory(WebRTCFactory):
self.force_sync = force_sync
self.release = release
self.AddCommonStep(['rm', '-rf', 'trunk'], descriptor='Cleanup',
warn_on_failure=True, halt_build_on_failure=False)
self.AddSmartCleanStep()
# Valgrind bots need special GYP defines to enable memory profiling
# friendly compilation. They already has a custom .gclient configuration
@ -525,8 +620,8 @@ class WebRTCLinuxFactory(WebRTCFactory):
class WebRTCMacFactory(WebRTCFactory):
"""Sets up the Mac build, both for make and xcode."""
def __init__(self):
WebRTCFactory.__init__(self)
def __init__(self, build_status_oracle):
WebRTCFactory.__init__(self, build_status_oracle)
self.build_type = 'both'
self.allowed_build_types = ['both', 'xcode', 'make']
@ -540,7 +635,7 @@ class WebRTCMacFactory(WebRTCFactory):
sys.exit(0)
else:
self.build_type = build_type
self.AddCommonStep(['rm', '-rf', 'trunk'], descriptor='Cleanup')
self.AddSmartCleanStep()
self.AddCommonStep(['gclient', 'config', SVN_LOCATION],
descriptor='gclient_config')
cmd = ['gclient', 'sync']
@ -591,8 +686,8 @@ class WebRTCWinFactory(WebRTCFactory):
Allows building with Debug, Release or both in sequence.
"""
def __init__(self):
WebRTCFactory.__init__(self)
def __init__(self, build_status_oracle):
WebRTCFactory.__init__(self, build_status_oracle)
self.configuration = 'Debug'
self.platform = 'x64'
self.allowed_platforms = ['x64', 'Win32']
@ -613,7 +708,7 @@ class WebRTCWinFactory(WebRTCFactory):
else:
self.configuration = configuration
if not build_only:
self.AddCommonStep(['rm', '-rf', 'trunk'], descriptor='Cleanup')
self.AddSmartCleanStep()
self.AddCommonStep(['gclient', 'config', SVN_LOCATION],
descriptor='gclient_config')
cmd = ['gclient', 'sync']