Libvpx buildbot waterfall.

I think I found a decent way to avoid duplicating the whole configuration for the libvpx waterfall. It's not perfect but it works. I still haven't figured out what the best way to manage the slaves would be however, since they will need to know which master to connect to, or they'll pick the default they're configured for in slaves.cfg, which is the WebRTC waterfall (can be overridden with the TESTING_MASTER property, but that's only to be used for development and testing.

BUG=None
TEST=Tested on local master and slaves.

Review URL: https://webrtc-codereview.appspot.com/595005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@2310 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
kjellander@webrtc.org 2012-05-29 06:23:42 +00:00
parent ca8d788362
commit fe87f8ce08
13 changed files with 173 additions and 94 deletions

41
tools/.gitignore vendored
View File

@ -18,13 +18,16 @@
/third_party/webtreemap
# Buildbot creates these files:
*.new
.manhole
bg_gradient.jpg
changes.pck
changes.pck.old
dropin.cache
events_*
http.log
master.cfg.sample
robots.txt
state.sqlite
state.sqlite-shm
state.sqlite-wal
@ -35,36 +38,8 @@ twistd.pid
/continuous_build/build
# Buildbot slave log directories:
/continuous_build/build_internal/masters/master.webrtc/android
/continuous_build/build_internal/masters/master.webrtc/android-ndk
/continuous_build/build_internal/masters/master.webrtc/chromeos
/continuous_build/build_internal/masters/master.webrtc/linux-asan
/continuous_build/build_internal/masters/master.webrtc/linux-clang
/continuous_build/build_internal/masters/master.webrtc/linux-large-tests
/continuous_build/build_internal/masters/master.webrtc/linux-memcheck
/continuous_build/build_internal/masters/master.webrtc/linux-slave-1
/continuous_build/build_internal/masters/master.webrtc/linux-slave-2
/continuous_build/build_internal/masters/master.webrtc/linux-slave-4
/continuous_build/build_internal/masters/master.webrtc/linux-slave-5
/continuous_build/build_internal/masters/master.webrtc/linux-slave-gcc-4.6
/continuous_build/build_internal/masters/master.webrtc/linux-tsan
/continuous_build/build_internal/masters/master.webrtc/LinuxChrome
/continuous_build/build_internal/masters/master.webrtc/LinuxChromeBloat
/continuous_build/build_internal/masters/master.webrtc/mac-large-tests/
/continuous_build/build_internal/masters/master.webrtc/mac-slave-2
/continuous_build/build_internal/masters/master.webrtc/mac-slave-3
/continuous_build/build_internal/masters/master.webrtc/MacChrome
/continuous_build/build_internal/masters/master.webrtc/public_html
/continuous_build/build_internal/masters/master.webrtc/win-32-dbg
/continuous_build/build_internal/masters/master.webrtc/win-32-release
/continuous_build/build_internal/masters/master.webrtc/WinChrome
/continuous_build/build_internal/masters/master.webrtc/win-large-tests
/continuous_build/build_internal/masters/master.tryserver.webrtc/android-trybot
/continuous_build/build_internal/masters/master.tryserver.webrtc/android-ndk-trybot
/continuous_build/build_internal/masters/master.tryserver.webrtc/linux-trybot-1
/continuous_build/build_internal/masters/master.tryserver.webrtc/linux-trybot-2
/continuous_build/build_internal/masters/master.tryserver.webrtc/mac-trybot-1
/continuous_build/build_internal/masters/master.tryserver.webrtc/mac-trybot-2
/continuous_build/build_internal/masters/master.tryserver.webrtc/public_html
/continuous_build/build_internal/masters/master.tryserver.webrtc/win-trybot-1
/continuous_build/build_internal/masters/master.tryserver.webrtc/win-trybot-2
/continuous_build/build_internal/masters/master.libvpx/*/
/continuous_build/build_internal/masters/master.webrtc/*/
/continuous_build/build_internal/masters/master.tryserver.webrtc/*/
# Public HTML folder is also a subdirectory, make sure it's not ignored:
!public_html

View File

@ -0,0 +1,5 @@
# -*- makefile -*-
BUILDBOT_PATH = $(BUILDBOT8_PATH)
MASTERPATH := master.libvpx
include ../master-common.mk

View File

@ -0,0 +1,4 @@
#!/usr/bin/python
from buildbot.scripts import runner
runner.run()

View File

@ -0,0 +1,11 @@
import os
from twisted.application import service
from buildbot.master import BuildMaster
basedir = os.path.dirname(os.path.abspath(__file__))
configfile = r'master.cfg'
application = service.Application('buildmaster')
BuildMaster(basedir, configfile).setServiceParent(application)

View File

@ -0,0 +1,26 @@
#!/usr/bin/env python
# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
__author__ = 'kjellander@webrtc.org (Henrik Kjellander)'
# This master configuration file reuses most of the WebRTC buildbot master,
# to avoid duplicating the configuration as much as possible.
# It uses a global variable to get the different configuration required for
# the libvpx source path to be set as custom_deps for the build factories.
LIBVPX_BUILD = True
LIBVPX_DEPS_LIST = [
('trunk/third_party/libvpx/source/libvpx',
'http://git.chromium.org/webm/libvpx.git'),
]
vars = {'LIBVPX_BUILD':LIBVPX_BUILD,
'LIBVPX_DEPS_LIST': LIBVPX_DEPS_LIST}
execfile('../master.webrtc/master.cfg', vars, vars)
c = BuildmasterConfig = vars['c']

View File

@ -0,0 +1 @@
../master.webrtc/master_builders_cfg.py

View File

@ -0,0 +1 @@
../master.webrtc/master_source_cfg.py

View File

@ -0,0 +1 @@
../master.webrtc/public_html

View File

@ -0,0 +1 @@
../master.webrtc/slaves.cfg

View File

@ -31,38 +31,50 @@ from webrtc_buildbot import utils
ActiveMaster = config.Master.WebRTC
def is_libvpx_build():
"""Used to check if this master is the default or the libvpx master."""
return 'LIBVPX_BUILD' in globals()
####### CHANGESOURCES
import master_source_cfg
master_source_cfg.ConfigureChangeSource(config, c)
####### SCHEDULERS
from buildbot.scheduler import Scheduler
webrtc_scheduler = Scheduler(name='all', branch='trunk',
builderNames=['Win32Debug',
'Win32Release',
'MacOS32DBG',
'MacOS32Release',
'Linux32DBG',
'Linux32Release',
'Linux64DBG',
'Linux64Release',
'LinuxClang',
'LinuxMemcheck',
'LinuxTsan',
'LinuxAsan',
'Linux64DBG-GCC4.6',
'LinuxLargeTests',
'MacLargeTests',
'WinLargeTests',
'Android',
'AndroidNDK',
'ChromeOS',
])
builder_names = ['Win32Debug',
'Win32Release',
'MacOS32DBG',
'MacOS32Release',
'Linux32DBG',
'Linux32Release',
'Linux64DBG',
'Linux64Release',
'LinuxClang',
'LinuxMemcheck',
'LinuxTsan',
'LinuxAsan',
'Linux64DBG-GCC4.6',
'LinuxLargeTests',
'MacLargeTests',
'WinLargeTests',
'Android',
'AndroidNDK',
'ChromeOS',
]
if is_libvpx_build():
# Run the libvpx waterfall only nightly since it re-uses the WebRTC bots and
# would cause colliding builds otherwise.
scheduler = timed.Nightly(name='libvpx_nightly', branch='trunk',
builderNames=builder_names, hour=20)
else:
# Trigger builds on each check-in if we are the regular WebRTC waterfall.
scheduler = Scheduler(name='all', branch='trunk', builderNames=builder_names)
# Note that additional schedulers (used by the Chrome+WebRTC builders) are
# defined in master_builders_cfg.py. This is an in progress change to make all
# our bots use the Chromium Buildbot scripts.
c['schedulers'] = [webrtc_scheduler]
c['schedulers'] = [scheduler]
####### TESTS
# Defines the supported tests followed by a tuple defining if the tests are
@ -156,31 +168,40 @@ CHROME_CUSTOM_DEPS_LIST = [
VALGRIND_DEPS_LIST = [
('trunk/third_party/valgrind',
'http://src.chromium.org/svn/trunk/deps/third_party/valgrind/binaries'),
'http://src.chromium.org/svn/trunk/deps/third_party/valgrind/binaries'),
]
ASAN_DEPS_LIST = [
('trunk/third_party/asan',
'http://src.chromium.org/chrome/trunk/deps/third_party/asan'),
]
DEFAULT_CUSTOM_DEPS = []
if is_libvpx_build():
# Additional configuration for libvpx waterfall (inherits this config).
DEFAULT_CUSTOM_DEPS += LIBVPX_DEPS_LIST
# Linux
linux_factory_64_dbg = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_factory_64_dbg'))
utils.BuildStatusOracle('linux_factory_64_dbg'),
custom_deps_list=DEFAULT_CUSTOM_DEPS)
linux_factory_64_dbg.EnableBuild()
linux_factory_64_dbg.EnableTests(linux_normal_tests)
linux_factory_32_release = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_factory_32_release'))
utils.BuildStatusOracle('linux_factory_32_release'),
custom_deps_list=DEFAULT_CUSTOM_DEPS)
linux_factory_32_release.EnableBuild(release=True, build32=True)
linux_factory_32_release.EnableTests(linux_normal_tests)
linux_factory_64_release = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_factory_64_release'))
utils.BuildStatusOracle('linux_factory_64_release'),
custom_deps_list=DEFAULT_CUSTOM_DEPS)
linux_factory_64_release.EnableBuild(release=True)
linux_factory_64_release.EnableTests(linux_normal_tests)
linux_factory_32_dbg = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_factory_32_dbg'))
utils.BuildStatusOracle('linux_factory_32_dbg'),
custom_deps_list=DEFAULT_CUSTOM_DEPS)
linux_factory_32_dbg.EnableCoverage(
coverage_url='http://webrtc-cb-linux-slave-4.cbf.corp.google.com/coverage/')
linux_factory_32_dbg.EnableBuild(build32=True)
@ -190,25 +211,27 @@ linux_factory_32_dbg.EnableTests(linux_normal_tests)
# under tooling. Not all tests will though since that would be too slow.
linux_factory_large_tests = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_factory_large_tests'),
custom_deps_list=VALGRIND_DEPS_LIST)
custom_deps_list=VALGRIND_DEPS_LIST + DEFAULT_CUSTOM_DEPS)
linux_factory_large_tests.EnableCoverage(
coverage_url='http://webrtc-build-bot-se.lul/coverage/')
linux_factory_large_tests.EnableBuild(compile_for_memory_tooling=True)
linux_factory_large_tests.EnableTests(linux_physical_machine_tests)
chromeos_factory = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('chromeos_factory'))
utils.BuildStatusOracle('chromeos_factory'),
custom_deps_list=DEFAULT_CUSTOM_DEPS)
chromeos_factory.EnableBuild(chrome_os=True)
chromeos_factory.EnableTests(linux_normal_tests)
linux_clang = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_clang'))
utils.BuildStatusOracle('linux_clang'),
custom_deps_list=DEFAULT_CUSTOM_DEPS)
linux_clang.EnableBuild(clang=True)
linux_clang.EnableTests(linux_normal_tests)
linux_memcheck_factory = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_memcheck_factory'), run_with_memcheck=True,
custom_deps_list=VALGRIND_DEPS_LIST)
custom_deps_list=VALGRIND_DEPS_LIST + DEFAULT_CUSTOM_DEPS)
linux_memcheck_factory.EnableBuild(release=True,
compile_for_memory_tooling=True)
# Filter out disabled memcheck tests:
@ -218,7 +241,7 @@ linux_memcheck_factory.EnableTests(memcheck_tests)
linux_tsan_factory = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_tsan_factory'), run_with_tsan=True,
custom_deps_list=VALGRIND_DEPS_LIST)
custom_deps_list=VALGRIND_DEPS_LIST + DEFAULT_CUSTOM_DEPS)
linux_tsan_factory.EnableBuild(release=True, compile_for_memory_tooling=True)
# Filter out disabled tsan tests:
tsan_tests = filter(lambda test: test not in TSAN_DISABLED_TESTS,
@ -227,7 +250,7 @@ linux_tsan_factory.EnableTests(tsan_tests)
linux_asan_factory = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_asan_factory'), run_with_asan=True,
custom_deps_list=ASAN_DEPS_LIST)
custom_deps_list=ASAN_DEPS_LIST + DEFAULT_CUSTOM_DEPS)
linux_asan_factory.EnableBuild(release=True, clang=True)
# Filter out disabled asan tests:
asan_tests = filter(lambda test: test not in ASAN_DISABLED_TESTS,
@ -235,42 +258,50 @@ asan_tests = filter(lambda test: test not in ASAN_DISABLED_TESTS,
linux_asan_factory.EnableTests(asan_tests)
android_factory = utils.WebRTCAndroidFactory(
utils.BuildStatusOracle('android_factory'))
utils.BuildStatusOracle('android_factory'),
custom_deps_list=DEFAULT_CUSTOM_DEPS)
android_factory.EnableBuild(product='toro')
android_ndk_factory = utils.WebRTCAndroidNDKFactory(
utils.BuildStatusOracle('android_ndk_factory'))
utils.BuildStatusOracle('android_ndk_factory'),
custom_deps_list=DEFAULT_CUSTOM_DEPS)
android_ndk_factory.EnableBuild()
# Mac
mac_factory_32_dbg = utils.WebRTCMacFactory(
utils.BuildStatusOracle('mac_factory_32_dbg'))
utils.BuildStatusOracle('mac_factory_32_dbg'),
custom_deps_list=DEFAULT_CUSTOM_DEPS)
mac_factory_32_dbg.EnableBuild(build_type='both')
mac_factory_32_dbg.EnableTests(mac_normal_tests)
mac_factory_32_release = utils.WebRTCMacFactory(
utils.BuildStatusOracle('mac_factory_32_release'))
utils.BuildStatusOracle('mac_factory_32_release'),
custom_deps_list=DEFAULT_CUSTOM_DEPS)
mac_factory_32_release.EnableBuild(build_type='both', release=True)
mac_factory_32_release.EnableTests(mac_normal_tests)
mac_factory_large_tests = utils.WebRTCMacFactory(
utils.BuildStatusOracle('mac_factory_large_tests'))
utils.BuildStatusOracle('mac_factory_large_tests'),
custom_deps_list=DEFAULT_CUSTOM_DEPS)
mac_factory_large_tests.EnableBuild(build_type='make')
mac_factory_large_tests.EnableTests(mac_physical_machine_tests)
# Windows
win_factory_32_Debug = utils.WebRTCWinFactory(
utils.BuildStatusOracle('win_factory_32_debug'))
utils.BuildStatusOracle('win_factory_32_debug'),
custom_deps_list=DEFAULT_CUSTOM_DEPS)
win_factory_32_Debug.EnableBuild(configuration='Debug')
win_factory_32_Debug.EnableTests(windows_normal_tests)
win_factory_32_Release = utils.WebRTCWinFactory(
utils.BuildStatusOracle('win_factory_32_release'))
utils.BuildStatusOracle('win_factory_32_release'),
custom_deps_list=DEFAULT_CUSTOM_DEPS)
win_factory_32_Release.EnableBuild(configuration='Release')
win_factory_32_Release.EnableTests(windows_normal_tests)
win_factory_large_tests = utils.WebRTCWinFactory(
utils.BuildStatusOracle('win_factory_large_tests'))
utils.BuildStatusOracle('win_factory_large_tests'),
custom_deps_list=DEFAULT_CUSTOM_DEPS)
win_factory_large_tests.EnableBuild()
win_factory_large_tests.EnableTests(windows_physical_machine_tests)
@ -414,8 +445,10 @@ c['builders'] = [
# Load Chrome bots that are using Chromium factories.
# These modules come from the local directory.
import master_builders_cfg
master_builders_cfg.ConfigureChromeWebRTCBuilders(c)
master_builders_cfg.ConfigureNightlyChromeWebRTCBloatBuilder(c)
master_builders_cfg.ConfigureChromeWebRTCBuilders(c,
custom_deps_list=DEFAULT_CUSTOM_DEPS)
master_builders_cfg.ConfigureNightlyChromeWebRTCBloatBuilder(c,
custom_deps_list=DEFAULT_CUSTOM_DEPS)
####### BUILDSLAVES

View File

@ -30,7 +30,7 @@ def win():
CHROME_LKGR = 'http://chromium-status.appspot.com/lkgr'
def ConfigureChromeWebRTCBuilders(c):
def ConfigureChromeWebRTCBuilders(c, custom_deps_list=[]):
helper = master_config.Helper(defaults)
B = helper.Builder
F = helper.Factory
@ -49,14 +49,16 @@ def ConfigureChromeWebRTCBuilders(c):
F('chrome_linux_debug_factory', linux().ChromiumWebRTCLatestFactory(
target='Debug',
factory_properties={'safesync_url': CHROME_LKGR,
'use_xvfb_on_linux': True}))
'use_xvfb_on_linux': True},
custom_deps_list=custom_deps_list))
# Mac 10.7 (Lion) ...
defaults['category'] = 'mac-10.7'
B('MacChrome', 'chrome_mac_debug_factory', scheduler='webrtc_rel')
F('chrome_mac_debug_factory', mac().ChromiumWebRTCLatestFactory(
target='Debug',
factory_properties={'safesync_url': CHROME_LKGR}))
factory_properties={'safesync_url': CHROME_LKGR},
custom_deps_list=custom_deps_list))
# Windows...
defaults['category'] = 'windows'
@ -64,14 +66,15 @@ def ConfigureChromeWebRTCBuilders(c):
F('chrome_win32_debug_factory', win().ChromiumWebRTCLatestFactory(
project=r'..\chrome\chrome.sln',
target='Debug',
factory_properties={'safesync_url': CHROME_LKGR}))
factory_properties={'safesync_url': CHROME_LKGR},
custom_deps_list=custom_deps_list))
# Use the helper class to connect the builders, factories and schedulers
# and add them to the BuildmasterConfig (c) dictionary.
helper.Update(c)
def ConfigureNightlyChromeWebRTCBloatBuilder(c):
def ConfigureNightlyChromeWebRTCBloatBuilder(c, custom_deps_list=[]):
# Nightly Scheduler at 22 PM CST/CDT. This will mean 5 AM in the CET
# time zone, which should avoid everyone's working hours.
nightly_scheduler = timed.Nightly(name='webrtc_nightly',
@ -85,7 +88,9 @@ def ConfigureNightlyChromeWebRTCBloatBuilder(c):
chrome_bloat_factory = linux().ChromiumWebRTCBloatFactory(
target='Release',
factory_properties={'safesync_url': CHROME_LKGR,
'gclient_env': {'GYP_DEFINES': 'profiling=1'}})
'gclient_env': {'GYP_DEFINES': 'profiling=1'}},
custom_deps_list=custom_deps_list)
chrome_bloat_builder = {
'name': 'LinuxChromeBloat',
'factory': chrome_bloat_factory,

View File

@ -501,8 +501,11 @@ class GenerateCodeCoverage(ShellCommand):
class WebRTCAndroidFactory(WebRTCFactory):
"""Sets up the Android build."""
def __init__(self, build_status_oracle, is_try_slave=False):
WebRTCFactory.__init__(self, build_status_oracle, is_try_slave)
def __init__(self, build_status_oracle, is_try_slave=False,
custom_deps_list=None):
WebRTCFactory.__init__(self, build_status_oracle=build_status_oracle,
is_try_slave=is_try_slave,
custom_deps_list=custom_deps_list)
def EnableBuild(self, product='toro'):
prefix = 'rm -rf out/target/product/%s/obj/' % product
@ -527,8 +530,11 @@ class WebRTCAndroidFactory(WebRTCFactory):
class WebRTCAndroidNDKFactory(WebRTCFactory):
"""Sets up the Android NDK build."""
def __init__(self, build_status_oracle, is_try_slave=False):
WebRTCFactory.__init__(self, build_status_oracle, is_try_slave)
def __init__(self, build_status_oracle, is_try_slave=False,
custom_deps_list=None):
WebRTCFactory.__init__(self, build_status_oracle=build_status_oracle,
is_try_slave=is_try_slave,
custom_deps_list=custom_deps_list)
def EnableBuild(self):
self.AddSmartCleanStep()
@ -771,8 +777,11 @@ class WebRTCLinuxFactory(WebRTCFactory):
class WebRTCMacFactory(WebRTCFactory):
"""Sets up the Mac build, both for make and xcode."""
def __init__(self, build_status_oracle, is_try_slave=False):
WebRTCFactory.__init__(self, build_status_oracle, is_try_slave)
def __init__(self, build_status_oracle, is_try_slave=False,
custom_deps_list=None):
WebRTCFactory.__init__(self, build_status_oracle=build_status_oracle,
is_try_slave=is_try_slave,
custom_deps_list=custom_deps_list)
self.build_type = 'both'
self.allowed_build_types = ['both', 'xcode', 'make']
@ -869,8 +878,11 @@ class WebRTCWinFactory(WebRTCFactory):
BUILD_CMD = r'C:\Windows\Microsoft.NET\Framework\v3.5\msbuild.exe'
VCAM_PATH = r'C:\Program Files (x86)\e2eSoft\VCam\VCamManager.exe'
def __init__(self, build_status_oracle, is_try_slave=False):
WebRTCFactory.__init__(self, build_status_oracle, is_try_slave)
def __init__(self, build_status_oracle, is_try_slave=False,
custom_deps_list=None):
WebRTCFactory.__init__(self, build_status_oracle=build_status_oracle,
is_try_slave=is_try_slave,
custom_deps_list=custom_deps_list)
self.configuration = 'Debug'
self.platform = 'x64'
self.allowed_platforms = ['x64', 'Win32']

View File

@ -27,8 +27,10 @@ class ChromiumWebRTCFactory(chromium_factory.ChromiumFactory):
tests=None, mode=None,
slave_type='BuilderTester', options=None,
compile_timeout=1200, build_url=None,
project=None, factory_properties=None):
self._solutions[0].custom_deps_list = [self.CUSTOM_DEPS_WEBRTC_LATEST]
project=None, factory_properties=None,
custom_deps_list=[]):
custom_deps_list.append(self.CUSTOM_DEPS_WEBRTC_LATEST)
self._solutions[0].custom_deps_list = custom_deps_list
factory = self.ChromiumFactory(target, clobber, tests, mode, slave_type,
options, compile_timeout, build_url, project,
factory_properties)
@ -43,8 +45,10 @@ class ChromiumWebRTCFactory(chromium_factory.ChromiumFactory):
tests=None, mode=None,
slave_type='BuilderTester', options=None,
compile_timeout=1200, build_url=None,
project=None, factory_properties=None):
self._solutions[0].custom_deps_list = [self.CUSTOM_DEPS_WEBRTC_LATEST]
project=None, factory_properties=None,
custom_deps_list=[]):
custom_deps_list.append(self.CUSTOM_DEPS_WEBRTC_LATEST)
self._solutions[0].custom_deps_list = custom_deps_list
factory = self.ChromiumFactory(target, clobber, tests, mode, slave_type,
options, compile_timeout, build_url, project,
factory_properties)