diff --git a/tools/.gitignore b/tools/.gitignore new file mode 100644 index 000000000..6aaedbeb7 --- /dev/null +++ b/tools/.gitignore @@ -0,0 +1,65 @@ +*.failed +*.pyc +*~ +.*.sw? +.DS_Store +.code_review_password +.cproject +.metadata +.project +.pydevproject +.settings +.status_password +/continuous_build/depot_tools +/third_party/gaeunit +/third_party/oauth2 + +# Buildbot creates these files: +.manhole +changes.pck +changes.pck.old +dropin.cache +events_* +http.log +master.cfg.sample +state.sqlite +state.sqlite-shm +state.sqlite-wal +twistd.log* +twistd.pid + +# Output by buildbot into public_html: +bg_gradient.jpg +default.css +favicon.ico.new +robots.txt + +# Chrome buildbot scripts +/continuous_build/build + +# Buildbot slave log directories: +/continuous_build/build_internal/masters/master.webrtc/android +/continuous_build/build_internal/masters/master.webrtc/chromeos +/continuous_build/build_internal/masters/master.webrtc/linux-chrome +/continuous_build/build_internal/masters/master.webrtc/linux-clang +/continuous_build/build_internal/masters/master.webrtc/linux-slave-1 +/continuous_build/build_internal/masters/master.webrtc/linux-slave-2 +/continuous_build/build_internal/masters/master.webrtc/linux-slave-4 +/continuous_build/build_internal/masters/master.webrtc/linux-slave-5 +/continuous_build/build_internal/masters/master.webrtc/linux-slave-gcc-4.6 +/continuous_build/build_internal/masters/master.webrtc/linux-valgrind +/continuous_build/build_internal/masters/master.webrtc/mac-slave-2 +/continuous_build/build_internal/masters/master.webrtc/mac-slave-3 +/continuous_build/build_internal/masters/master.webrtc/win-32-dbg +/continuous_build/build_internal/masters/master.webrtc/win-32-release +/continuous_build/build_internal/masters/master.webrtc/video +/continuous_build/build_internal/masters/master.tryserver.webrtc/linux-trybot-1 +/continuous_build/build_internal/masters/master.tryserver.webrtc/linux-trybot-2 +/continuous_build/build_internal/masters/master.tryserver.webrtc/mac-trybot-1 +/continuous_build/build_internal/masters/master.tryserver.webrtc/mac-trybot-2 +/continuous_build/build_internal/masters/master.tryserver.webrtc/win-trybot-1 +/continuous_build/build_internal/masters/master.tryserver.webrtc/win-trybot-2 + + + + diff --git a/tools/DEPS b/tools/DEPS new file mode 100644 index 000000000..68dd43abf --- /dev/null +++ b/tools/DEPS @@ -0,0 +1,34 @@ +# Tools has its own dependencies, separate from the production code. +# Use http rather than https; the latter can cause problems for users behind +# proxies. + +vars = { + "chromium_trunk" : "http://src.chromium.org/svn/trunk", +} + +deps = { + # Used by quality_tracking. + "tools/third_party/gaeunit": + "http://code.google.com/p/gaeunit.git@e16d5bd4", + + # Used by quality_tracking. + "tools/third_party/oauth2": + "http://github.com/simplegeo/python-oauth2.git@a83f4a29", + + # Chromium buildbot scripts needs to be at the same level as our scripts. + "tools/continuous_build/build": + Var("chromium_trunk") + "/tools/build", + + # Chromium buildbot scripts needs their own depot_tools. + "tools/continuous_build/depot_tools": + Var("chromium_trunk") + "/tools/depot_tools", +} + +hooks = [ + { + # Update slave buildbot.tac to include WebRTC slave_utils import. + "pattern": ".", + "action": ["python", "tools/add_webrtc_slave_utils.py"], + }, +] + diff --git a/tools/OWNERS b/tools/OWNERS new file mode 100644 index 000000000..965de1e93 --- /dev/null +++ b/tools/OWNERS @@ -0,0 +1,4 @@ +kjellander@webrtc.org +phoglund@webrtc.org +niklas.enbom@webrtc.org +andrew@webrtc.org diff --git a/tools/PRESUBMIT.py b/tools/PRESUBMIT.py new file mode 100644 index 000000000..5e2faaf0a --- /dev/null +++ b/tools/PRESUBMIT.py @@ -0,0 +1,58 @@ +# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +def _LicenseHeader(input_api): + """Returns the license header regexp.""" + license_header = ( + r'.*? Copyright \(c\) %(year)s The WebRTC project authors\. ' + r'All Rights Reserved\.\n' + r'.*?\n' + r'.*? Use of this source code is governed by a BSD-style license\n' + r'.*? that can be found in the LICENSE file in the root of the source\n' + r'.*? tree\. An additional intellectual property rights grant can be ' + r'found\n' + r'.*? in the file PATENTS\. All contributing project authors may\n' + r'.*? be found in the AUTHORS file in the root of the source tree\.\n' + ) % { + 'year': input_api.time.strftime('%Y'), + } + return license_header + +def _CommonChecks(input_api, output_api): + """Checks common to both upload and commit.""" + results = [] + results.extend(input_api.canned_checks.CheckLongLines( + input_api, output_api)) + results.extend(input_api.canned_checks.CheckChangeHasNoTabs( + input_api, output_api)) + results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace( + input_api, output_api)) + results.extend(input_api.canned_checks.CheckChangeTodoHasOwner( + input_api, output_api)) + results.extend(input_api.canned_checks.CheckLicense( + input_api, output_api, _LicenseHeader(input_api))) + return results + +def CheckChangeOnUpload(input_api, output_api): + results = [] + results.extend(_CommonChecks(input_api, output_api)) + return results + +def CheckChangeOnCommit(input_api, output_api): + results = [] + results.extend(_CommonChecks(input_api, output_api)) + results.extend(input_api.canned_checks.CheckOwners(input_api, output_api)) + results.extend(input_api.canned_checks.CheckChangeWasUploaded( + input_api, output_api)) + results.extend(input_api.canned_checks.CheckChangeHasDescription( + input_api, output_api)) + results.extend(input_api.canned_checks.CheckChangeHasBugField( + input_api, output_api)) + results.extend(input_api.canned_checks.CheckChangeHasTestField( + input_api, output_api)) + return results diff --git a/tools/add_webrtc_slave_utils.py b/tools/add_webrtc_slave_utils.py new file mode 100755 index 000000000..4b28ac6e8 --- /dev/null +++ b/tools/add_webrtc_slave_utils.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +__author__ = 'kjellander@webrtc.org (Henrik Kjellander)' + +import os + + +def main(): + """ + Simple script for adding an import of the WebRTC slave_utils module for the + buildbot slaves to the Chromium buildbot.tac file. + Using this script, we don't need to maintain our own version of the slave + scripts and can automatically stay up to date with their changes. + It will add a comment and the import at the end of the file, if it's not + already present. + + This script should be invoked as a hooks step in the DEPS file, like this: + hooks = [ + { + # Update slave buildbot.tac to include WebRTC slave_utils import. + "pattern": ".", + "action": ["python", "tools/add_webrtc_slave_utils.py"], + }, + ] + """ + SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) + TARGET_FILE = os.path.join(SCRIPT_PATH, + 'continuous_build/build/slave/buildbot.tac') + COMMENT_LINE = '# Load WebRTC custom slave script.\n' + IMPORT_LINE = 'from webrtc_buildbot import slave_utils\n' + + file = open(TARGET_FILE, 'r') + if file.read().find(IMPORT_LINE) == -1: + print 'Patching %s with WebRTC imports.' % TARGET_FILE + file.close() + file = open(TARGET_FILE, 'a') + file.write(COMMENT_LINE) + file.write(IMPORT_LINE) + file.close() + +if __name__ == '__main__': + main() diff --git a/tools/codereview.settings b/tools/codereview.settings new file mode 100644 index 000000000..c7ae7860d --- /dev/null +++ b/tools/codereview.settings @@ -0,0 +1,9 @@ +# This file is used by gcl to get repository specific information. +CODE_REVIEW_SERVER: webrtc-codereview.appspot.com +#CC_LIST: +#VIEW_VC: +#STATUS: +TRY_ON_UPLOAD: False +#TRYSERVER_SVN_URL: +#GITCL_PREUPLOAD: +#GITCL_PREDCOMMIT: diff --git a/tools/continuous_build/OWNERS b/tools/continuous_build/OWNERS index 60e166ca0..311c3c5cf 100644 --- a/tools/continuous_build/OWNERS +++ b/tools/continuous_build/OWNERS @@ -1,3 +1,3 @@ -ivinnichenko@webrtc.org -phoglund@webrtc.org kjellander@webrtc.org +phoglund@webrtc.org +ivinnichenko@webrtc.org diff --git a/tools/continuous_build/README b/tools/continuous_build/README new file mode 100644 index 000000000..9f48a7c92 --- /dev/null +++ b/tools/continuous_build/README @@ -0,0 +1,76 @@ +WebRTC buildbots +================ +These buildbot setups are based on how Chromium uses Buildbot scripts with their +own customizations of startup and master-slave configurations. + +The directory layout and the files inside build_internal/masters are copied from +Chromium's repository and modified for our needs according to the guidelines at +http://goo.gl/kPSSv + +One nice advantage with using this, is that all slaves run out of the box by +just syncing the WebRTC tools and the third-party dependencies. No additional +dependencies are required except a Python install. + + +Setup a tools workspace: +======================== +mkdir webrtc-tools +cd webrtc-tools +gclient config http://webrtc.googlecode.com/svn/trunk/tools +gclient sync +cd tools/continuous_build +svn co http://src.chromium.org/svn/trunk/tools/depot_tools +echo pass > build/site_config/.bot_password + + +To run the build master: +======================== +cd tools/continuous_build/build_internal/masters/master.webrtc +make restart + + +To run the try master: +====================== +cd tools/continuous_build/build_internal/masters/master.tryserver.webrtc +make restart + + +To run a slave: +=============== +cd tools/continuous_build/build/slave + +There are some variables that can be set for flexible development testing: +* TESTING_SLAVENAME : Simulates the slave name. Since slave names are + based on hostnames, they must be overridden using + the this variable in order to run the slave on the + local machine. +* TESTING_MASTER : The master to connect to. We usually use 'WebRTC' + or 'TryServer'. +* TESTING_MASTER_HOST : The host the master is located at. + Default: localhost +* RUN_SLAVE_UPDATED_SCRIPTS : Set this to False to avoid checking for updates + during startup. + +For Windows, make sure to add the --use_buildbot_8 flag to the run_slave.bat +in order to make the tests execute properly. + +Examples: +--------- + +Connect to a running build master: +* Linux and Mac: +TESTING_SLAVENAME=webrtc-cb-linux-slave-1 TESTING_MASTER=WebRTC make restart + +* Windows: +set TESTING_SLAVENAME=webrtc-cb-linux-slave-1 +set TESTING_MASTER=WebRTC +run_slave.bat --use_buildbot_8 + +Connect to a running try master: +* Linux and Mac: +TESTING_SLAVENAME=linux-trybot-1 TESTING_MASTER=TryServer make restart + +* Windows: +set TESTING_SLAVENAME=linux-trybot-1 +set TESTING_MASTER=TryServer +run_slave.bat --use_buildbot_8 diff --git a/tools/continuous_build/build_internal/masters/master-common.mk b/tools/continuous_build/build_internal/masters/master-common.mk new file mode 100644 index 000000000..7a5288f9c --- /dev/null +++ b/tools/continuous_build/build_internal/masters/master-common.mk @@ -0,0 +1,123 @@ +# -*- makefile -*- + +# This file is a copy of the Chromium master-common.mk located at +# /trunk/tools/build/masters of the Chromium tools. It is just modified to add +# the PRIVATESCRIPTS_DIR property. + +# This should be included by a makefile which lives in a buildmaster/buildslave +# directory (next to the buildbot.tac file). That including makefile *must* +# define MASTERPATH. + +# The 'start' and 'stop' targets start and stop the buildbot master. +# The 'reconfig' target will tell a buildmaster to reload its config file. + +# Note that a relative PYTHONPATH entry is relative to the current directory. + +# Confirm that MASTERPATH has been defined. +ifeq ($(MASTERPATH),) + $(error MASTERPATH not defined.) +endif + +# On the Mac, the buildbot is started via the launchd mechanism as a +# LaunchAgent to give the slave a proper Mac UI environment for tests. In +# order for this to work, the plist must be present and loaded by launchd, and +# the user must be logged in to the UI. The plist is loaded by launchd at user +# login (and the job may have been initially started at that time too). Our +# Mac build slaves are all set up this way, and have auto-login enabled, so +# "make start" should just work and do the right thing. +# +# When using launchd to start the job, it also needs to be used to stop the +# job. Otherwise, launchd might try to restart the job when stopped manually +# by SIGTERM. Using SIGHUP for reconfig is safe with launchd. +# +# Because it's possible to have more than one slave on a machine (for testing), +# this tests to make sure that the slave is in the known slave location, +# /b/slave, which is what the LaunchAgent operates on. +USE_LAUNCHD := \ + $(shell [ -f ~/Library/LaunchAgents/org.chromium.buildbot.$(MASTERPATH).plist ] && \ + [ "$$(pwd -P)" = "/b/build/masters/$(MASTERPATH)" ] && \ + echo 1) + +# Elements used to construct PYTHONPATH. These may be overridden by the +# including Makefile. +# +# For example: while we transition from buildbot 0.7.12 to buildbot 0.8.x , +# some masters will override BUILDBOT_PATH in their local Makefiles. +TOPLEVEL_DIR ?= ../../../build +THIRDPARTY_DIR ?= $(TOPLEVEL_DIR)/third_party +SCRIPTS_DIR ?= $(TOPLEVEL_DIR)/scripts +PUBLICCONFIG_DIR ?= $(TOPLEVEL_DIR)/site_config +PRIVATECONFIG_DIR ?= $(TOPLEVEL_DIR)/../build_internal/site_config +PRIVATESCRIPTS_DIR ?= $(TOPLEVEL_DIR)/../build_internal/scripts + +# Packages needed by buildbot7 +BUILDBOT7_PATH = $(THIRDPARTY_DIR)/buildbot_7_12:$(THIRDPARTY_DIR)/twisted_8_1 + +# Packages needed by buildbot8 +BUILDBOT8_DEPS := \ + buildbot_8_4p1 \ + twisted_10_2 \ + jinja2 \ + sqlalchemy_0_7_1 \ + sqlalchemy_migrate_0_7_1 \ + tempita_0_5 \ + decorator_3_3_1 + +nullstring := +space := $(nullstring) # +BUILDBOT8_PATH = $(subst $(space),:,$(BUILDBOT8_DEPS:%=$(THIRDPARTY_DIR)/%)) + +# Default to buildbot7. To override, put this in the master's Makefile: +# BUILDBOT_PATH = $(BUILDBOT8_PATH) +BUILDBOT_PATH ?= $(BUILDBOT7_PATH) + +PYTHONPATH := $(BUILDBOT_PATH):$(SCRIPTS_DIR):$(THIRDPARTY_DIR):$(PUBLICCONFIG_DIR):$(PRIVATECONFIG_DIR):$(PRIVATESCRIPTS_DIR):. + +ifeq ($(BUILDBOT_PATH),$(BUILDBOT8_PATH)) +start: upgrade +else +start: +endif +ifneq ($(USE_LAUNCHD),1) + PYTHONPATH=$(PYTHONPATH) python $(SCRIPTS_DIR)/common/twistd --no_save -y buildbot.tac +else + launchctl start org.chromium.buildbot.$(MASTERPATH) +endif + +ifeq ($(BUILDBOT_PATH),$(BUILDBOT8_PATH)) +start-prof: upgrade +else +start-prof: +endif +ifneq ($(USE_LAUNCHD),1) + TWISTD_PROFILE=1 PYTHONPATH=$(PYTHONPATH) python $(SCRIPTS_DIR)/common/twistd --no_save -y buildbot.tac +else + launchctl start org.chromium.buildbot.$(MASTERPATH) +endif + +stop: +ifneq ($(USE_LAUNCHD),1) + if `test -f twistd.pid`; then kill `cat twistd.pid`; fi; +else + launchctl stop org.chromium.buildbot.$(MASTERPATH) +endif + +reconfig: + kill -HUP `cat twistd.pid` + +log: + tail -F twistd.log + +wait: + while `test -f twistd.pid`; do sleep 1; done; + +restart: stop wait start log + +restart-prof: stop wait start-prof log + +# This target is only known to work on 0.8.x masters. +upgrade: + @[ -e '.dbconfig' ] || [ -e 'state.sqlite' ] || PYTHONPATH=$(PYTHONPATH) python buildbot upgrade-master . + +setup: + @echo export PYTHONPATH=$(PYTHONPATH) diff --git a/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/Makefile b/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/Makefile new file mode 100644 index 000000000..53c040078 --- /dev/null +++ b/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/Makefile @@ -0,0 +1,5 @@ +# -*- makefile -*- + +BUILDBOT_PATH = $(BUILDBOT8_PATH) +MASTERPATH := master.tryserver.webrtc +include ../master-common.mk diff --git a/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/buildbot b/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/buildbot new file mode 100644 index 000000000..cf3628dd5 --- /dev/null +++ b/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/buildbot @@ -0,0 +1,4 @@ +#!/usr/bin/python + +from buildbot.scripts import runner +runner.run() diff --git a/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/buildbot.tac b/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/buildbot.tac new file mode 100644 index 000000000..f4bca882a --- /dev/null +++ b/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/buildbot.tac @@ -0,0 +1,11 @@ +import os + +from twisted.application import service +from buildbot.master import BuildMaster + +basedir = os.path.dirname(os.path.abspath(__file__)) +configfile = r'master.cfg' + +application = service.Application('buildmaster') +BuildMaster(basedir, configfile).setServiceParent(application) + diff --git a/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/master.cfg b/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/master.cfg new file mode 100755 index 000000000..f94b32c9a --- /dev/null +++ b/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/master.cfg @@ -0,0 +1,265 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +__author__ = 'kjellander@webrtc.org (Henrik Kjellander)' + +c = BuildmasterConfig = {} + +import os +from buildbot.buildslave import BuildSlave +from buildbot.changes.pb import PBChangeSource +from buildbot.changes.svnpoller import SVNPoller +from buildbot.process import factory +from buildbot.scheduler import Scheduler +from buildbot.status import html +from buildbot.status import mail +from buildbot.steps import shell + +# These modules come from scripts, which must be in the PYTHONPATH. +from master import master_utils +from master import slaves_list +from master.builders_pools import BuildersPools +from master.try_job_http import TryJobHTTP +from master.try_job_svn import TryJobSubversion +from master.try_mail_notifier import TryMailNotifier +import config + +from webrtc_buildbot import utils + +ActiveMaster = config.Master.TryServer + +####### CHANGESOURCES +c['change_source'] = [] + +####### SCHEDULERS +pools = BuildersPools('webrtc') +pools['webrtc'].append('linux') +pools['webrtc'].append('linux_rel') +pools['webrtc'].append('mac') +pools['webrtc'].append('mac_rel') +pools['webrtc'].append('win') +pools['webrtc'].append('win_rel') + +last_good_urls = {'webrtc': ActiveMaster.last_good_url} +code_review_sites = {'webrtc': ActiveMaster.code_review_site} + +try_scheduler = TryJobHTTP(name='try_job_http', + port=ActiveMaster.try_job_port, + last_good_urls=last_good_urls, + code_review_sites=code_review_sites, + pools=pools) +c['schedulers'] = [try_scheduler] + +####### TESTS + +# Tests to run on the trybots +# Defines the supported tests followed by a tuple defining if the tests are +# enabled on Linux, Mac and/or Windows (in that order; defined in utils.py). +NORMAL_TESTS = { + # Test name Linux Mac Windows + 'audio_coding_unittests': (True, True, True), + 'audio_conference_mixer_unittests':(True, True, True), + 'audioproc_unittest': (True, True, True), + 'cng_unittests': (True, True, True), + 'g711_unittests': (True, True, True), + 'g722_unittests': (True, True, True), + 'libyuv_unittests': (True, True, True), + 'jpeg_unittests': (True, True, True), + 'media_file_unittests': (True, True, True), + 'metrics_unittests': (True, True, True), + 'neteq_unittests': (True, True, True), + 'pcm16b_unittests': (True, True, True), + 'resampler_unittests': (True, True, True), + 'rtp_rtcp_unittests': (True, True, True), + 'signal_processing_unittests': (True, True, True), + 'system_wrappers_unittests': (True, True, True), + 'test_support_unittests': (True, True, True), + 'udp_transport_unittests': (True, True, True), + 'vad_unittests': (True, True, True), + 'video_codecs_test_framework_unittests': (True, True, True), + 'video_coding_unittests': (True, True, True), + 'video_engine_core_unittests': (True, True, True), + 'video_processing_unittests': (True, True, True), + 'voice_engine_unittests': (True, True, True), + 'vp8_unittests': (True, False, True), # Issue 273. + 'webrtc_utility_unittests': (True, True, False), +} +linux_normal_tests = utils.GetEnabledTests(NORMAL_TESTS, 'Linux') +mac_normal_tests = utils.GetEnabledTests(NORMAL_TESTS, 'Mac') +windows_normal_tests = utils.GetEnabledTests(NORMAL_TESTS, 'Windows') + +####### FACTORIES +# Linux +linux_factory_dbg = utils.WebRTCLinuxFactory( + utils.BuildStatusOracle('linux_factory_dbg'), is_try_slave=True) +linux_factory_dbg.EnableBuild() +linux_factory_dbg.EnableTests(linux_normal_tests) + +linux_factory_release = utils.WebRTCLinuxFactory( + utils.BuildStatusOracle('linux_factory_release'), is_try_slave=True) +linux_factory_release.EnableBuild() +linux_factory_release.EnableTests(linux_normal_tests) + +# Mac +mac_factory_dbg = utils.WebRTCMacFactory( + utils.BuildStatusOracle('mac_factory_dbg'), is_try_slave=True) +mac_factory_dbg.EnableBuild(build_type='make') +mac_factory_dbg.EnableTests(mac_normal_tests) + +mac_factory_release = utils.WebRTCMacFactory( + utils.BuildStatusOracle('mac_factory_release'), is_try_slave=True) +mac_factory_release.EnableBuild(build_type='make') +mac_factory_release.EnableTests(mac_normal_tests) + +# Windows +win_factory_debug = utils.WebRTCWinFactory( + utils.BuildStatusOracle('win_factory_debug'), is_try_slave=True) +win_factory_debug.EnableBuild(configuration='Debug') +win_factory_debug.EnableTests(windows_normal_tests) + +win_factory_release = utils.WebRTCWinFactory( + utils.BuildStatusOracle('win_factory_release'), is_try_slave=True) +win_factory_release.EnableBuild(configuration='Release') +win_factory_release.EnableTests(windows_normal_tests) + +####### BUILDERS +linux_builder_debug = { + 'name': 'linux', + 'slavename': 'webrtc-cb-linux-slave-11', + 'builddir': 'linux-trybot-1', + 'factory': linux_factory_dbg, + } +linux_builder_release = { + 'name': 'linux_rel', + 'slavename': 'webrtc-cb-linux-slave-12', + 'builddir': 'linux-trybot-2', + 'factory': linux_factory_release, + } +mac_builder_debug = { + 'name': 'mac', + 'slavename': 'dhcp-172-28-249-242', + 'builddir': 'mac-trybot-1', + 'factory': mac_factory_dbg, + } +mac_builder_release = { + 'name': 'mac_rel', + 'slavename': 'dhcp-172-28-249-244', + 'builddir': 'mac-trybot-2', + 'factory': mac_factory_release, + } +win_builder_debug = { + 'name': 'win', + 'slavename': 'webrtc-win-x64', + 'builddir': 'win-trybot-1', + 'factory': win_factory_debug, + } +win_builder_release = { + 'name': 'win_rel', + 'slavename': 'webrtc-win-x86', + 'builddir': 'win-trybot-2', + 'factory': win_factory_release, + } + +c['builders'] = [ + win_builder_debug, + win_builder_release, + mac_builder_debug, + mac_builder_release, + linux_builder_debug, + linux_builder_release, + ] + + +def NextJob(builder, requests): + """Always prioritize commit queue jobs over try jobs.""" + for req in requests: + if any(c.who == 'commit-bot@webrtc.org' for c in req.source.changes): + return req + return requests[0] + + +# Slaves are loaded from slaves.cfg. +slaves = slaves_list.SlavesList('slaves.cfg', 'TryServer') + +for builder in c['builders']: + # Associate the slaves to the builders. The configuration is in slaves.cfg. + builder['slavenames'] = slaves.GetSlavesName(builder=builder['name']) + # Don't enable auto_reboot for our bots (Chrome uses it since they don't trust + # their unit tests to clean up properly) + builder['auto_reboot'] = False + # Prioritize commit-queue jobs over try jobs. + # This is documented starting with 0.8.2 + # http://buildbot.net/buildbot/docs/0.8.2/Prioritizing-Builds.html#Prioritizing-Builds + # but appears to be working on 0.7.12 already. + builder['nextBuild'] = NextJob + + +####### BUILDSLAVES + +# The 'slaves' list defines the set of allowable buildslaves. List all the +# slaves registered to a builder. Remove dupes. +c['slaves'] = master_utils.AutoSetupSlaves(c['builders'], + config.Master.GetBotPassword()) + +# Make sure everything works together. +master_utils.VerifySetup(c, slaves) + +# Adds common status and tools to this master. +# Use our own mail notifier. +master_utils.AutoSetupMaster(c, ActiveMaster, False, + public_html='../master.webrtc/public_html', + templates=['./templates', + '../master.webrtc/templates']) + +####### STATUS TARGETS +# Port 9010 is the same as Chromium's try server web interface: +web_page = html.WebStatus(http_port=9010, allowForce=True) +c['status'] = [web_page] + +# Use an environment variable to easily avoid enabling e-mail for development. +if not os.getenv('BUILDBOT_DEVELOPMENT_MODE'): + email_status = mail.MailNotifier( + fromaddr='webrtc-cb-watchlist@google.com', + extraRecipients=['webrtc-cb-watchlist@google.com'], + sendToInterestedUsers=True, + mode='failing') + c['status'] += [email_status] + +# Try job result emails. +c['status'].append(TryMailNotifier( + fromaddr='webrtc-cb-watchlist@google.com', + subject='try %(result)s for %(reason)s on %(builder)s @ r%(revision)s', + mode='all', + lookup=master_utils.UsersAreEmails())) + +####### DB URL + +# This specifies what database buildbot uses to store change and scheduler +# state. You can leave this at its default for all but the largest +# installations. +c['db_url'] = 'sqlite:///state.sqlite' + +####### DEBUGGING OPTIONS + +# if you set 'debugPassword', then you can connect to the buildmaster with +# the diagnostic tool in contrib/debugclient.py . From this tool, you can +# manually force builds and inject changes, which may be useful for testing +# your buildmaster without actually committing changes to your repository (or +# before you have a functioning 'sources' set up). The debug tool uses the +# same port number as the slaves do: 'slavePortnum'. + +#c['debugPassword'] = 'debugpassword' + +# if you set 'manhole', you can ssh into the buildmaster and get an +# interactive python shell, which may be useful for debugging buildbot +# internals. It is probably only useful for buildbot developers. You can also +# use an authorized_keys file, or plain telnet. +#from buildbot import manhole +#c['manhole'] = manhole.PasswordManhole('tcp:9999:interface=127.0.0.1', +# 'admin', 'password') diff --git a/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/public_html/buildbot.css b/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/public_html/buildbot.css new file mode 100644 index 000000000..1fb556d3a --- /dev/null +++ b/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/public_html/buildbot.css @@ -0,0 +1,457 @@ +body { + margin-bottom:50px; +} + +body, td { + font-family: Verdana, Cursor; + font-size: 10px; + font-weight: bold; +} + +a:link,a:visited,a:active { + color: #444; +} + +a:hover { + color: #000000; +} + +table { + border-spacing: 1px 1px; +} + +table td { + padding: 3px 0px 3px 0px; + text-align: center; +} + +.Project { + width: 100px; +} + +.LastBuild, .Activity { + padding: 0 0 0 4px; +} + +.LastBuild, .Activity, .Builder, .BuildStep { + width: 155px; + max-width: 155px; +} + +/* Chromium Specific styles */ +div.BuildResultInfo { + color: #444; +} + +div.Announcement { + margin-bottom: 1em; +} + +div.Announcement > a:hover { + color: black; +} + +div.Announcement > div.Notice { + background-color: #afdaff; + padding: 0.5em; + font-size: 16px; + text-align: center; +} + +div.Announcement > div.Open { + border: 3px solid #8fdf5f; + padding: 0.5em; + font-size: 16px; + text-align: center; +} + +div.Announcement > div.Closed { + border: 5px solid #e98080; + padding: 0.5em; + font-size: 24px; + font-weight: bold; + text-align: center; +} + +td.Time { + color: #000; + border-bottom: 1px solid #aaa; + background-color: #eee; +} + +td.Activity, td.Change, td.Builder { + color: #333333; + background-color: #CCCCCC; +} + +td.Change { + border-radius: 5px; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; +} +td.Event { + color: #777; + background-color: #ddd; + border-radius: 5px; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; +} + +td.Activity { + border-top-left-radius: 10px; + -webkit-border-top-left-radius: 10px; + -moz-border-radius-topleft: 10px; + min-height: 20px; + padding: 2px 0 2px 0; +} + +td.idle, td.waiting, td.offline, td.building { + border-top-left-radius: 0px; + -webkit-border-top-left-radius: 0px; + -moz-border-radius-topleft: 0px; +} + +.LastBuild { + border-top-left-radius: 5px; + -webkit-border-top-left-radius: 5px; + -moz-border-radius-topleft: 5px; + border-top-right-radius: 5px; + -webkit-border-top-right-radius: 5px; + -moz-border-radius-topright: 5px; +} + +/* Console view styles */ + +td.DevRev { + padding: 4px 8px 4px 8px; + color: #333333; + border-top-left-radius: 5px; + -webkit-border-top-left-radius: 5px; + -moz-border-radius-topleft: 5px; + background-color: #eee; + width: 1%; +} + +td.DevRevCollapse { + border-bottom-left-radius: 5px; + -webkit-border-bottom-left-radius: 5px; + -moz-border-radius-bottomleft: 5px; +} + +td.DevName { + padding: 4px 8px 4px 8px; + color: #333333; + background-color: #eee; + width: 1%; + text-align: left; +} + +td.DevStatus { + padding: 4px 4px 4px 4px; + color: #333333; + background-color: #eee; +} + +td.DevSlave { + padding: 4px 4px 4px 4px; + color: #333333; + background-color: #eee; +} + +td.first { + border-top-left-radius: 5px; + -webkit-border-top-left-radius: 5px; + -moz-border-radius-topleft: 5px; +} + +td.last { + border-top-right-radius: 5px; + -webkit-border-top-right-radius: 5px; + -moz-border-radius-topright: 5px; +} + +td.DevStatusCategory { + border-radius: 5px; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; + border-width:1px; + border-style:solid; +} + +td.DevStatusCollapse { + border-bottom-right-radius: 5px; + -webkit-border-bottom-right-radius: 5px; + -moz-border-radius-bottomright: 5px; +} + +td.DevDetails { + font-weight: normal; + padding: 8px 8px 8px 8px; + color: #333333; + background-color: #eee; + text-align: left; +} + +td.DevComment { + font-weight: normal; + padding: 8px 8px 8px 8px; + color: #333333; + border-bottom-right-radius: 5px; + -webkit-border-bottom-right-radius: 5px; + -moz-border-radius-bottomright: 5px; + border-bottom-left-radius: 5px; + -webkit-border-bottom-left-radius: 5px; + -moz-border-radius-bottomleft: 5px; + background-color: #eee; + text-align: left; +} + +td.Alt { + background-color: #CCCCCC; +} + +.legend { + border-radius: 5px; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; + width: 100px; + max-width: 100px; + text-align:center; + padding: 2px 2px 2px 2px; + height:14px; + white-space:nowrap; +} + +.DevStatusBox { + text-align:center; + height:20px; + padding:0 2px; + line-height:0; + white-space:nowrap; +} + +.DevStatusBox a { + opacity: 0.85; + border-width:1px; + border-style:solid; + border-radius: 4px; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + display:block; + width:90%; + height:20px; + line-height:20px; + margin-left: auto; + margin-right: auto; +} + +.DevSlaveBox { + text-align:center; + height:10px; + padding:0 2px; + line-height:0; + white-space:nowrap; +} + +.DevSlaveBox a { + opacity: 0.85; + border-width:1px; + border-style:solid; + border-radius: 4px; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + display:block; + width:90%; + height:10px; + line-height:20px; + margin-left: auto; + margin-right: auto; +} + +a.noround { + border-radius: 0px; + -webkit-border-radius: 0px; + -moz-border-radius: 0px; + position: relative; + margin-top: -8px; + margin-bottom: -8px; + height: 36px; + border-top-width: 0; + border-bottom-width: 0; +} + +a.begin { + border-top-width:1px; + position: relative; + margin-top: 0px; + margin-bottom: -7px; + height: 27px; + border-top-left-radius: 4px; + -webkit-border-top-left-radius: 4px; + -moz-border-radius-topleft: 4px; + border-top-right-radius: 4px; + -webkit-border-top-right-radius: 4px; + -moz-border-radius-topright: 4px; +} + +a.end { + border-bottom-width:1px; + position: relative; + margin-top: -7px; + margin-bottom: 0px; + height: 27px; + border-bottom-left-radius: 4px; + -webkit-border-bottom-left-radius: 4px; + -moz-border-radius-bottomleft: 4px; + border-bottom-right-radius: 4px; + -webkit-border-bottom-right-radius: 4px; + -moz-border-radius-bottomright: 4px; +} + +.center_align { + text-align: center; +} + +.right_align { + text-align: right; +} + +.left_align { + text-align: left; +} + +div.BuildWaterfall { + border-radius: 7px; + -webkit-border-radius: 7px; + -moz-border-radius: 7px; + position: absolute; + left: 0px; + top: 0px; + background-color: #FFFFFF; + padding: 4px 4px 4px 4px; + float: left; + display: none; + border-width: 1px; + border-style: solid; +} + +/* LastBuild, BuildStep states */ +.success { + color: #FFFFFF; + background-color: #8fdf5f; + border-color: #4F8530; +} + +.failure { + color: #FFFFFF; + background-color: #e98080; + border-color: #A77272; +} + +.warnings { + color: #FFFFFF; + background-color: #ffc343; + border-color: #C29D46; +} + +.exception, td.offline { + color: #FFFFFF; + background-color: #e0b0ff; + border-color: #ACA0B3; +} + +.start,.running, td.building { + color: #666666; + background-color: #fffc6c; + border-color: #C5C56D; +} + +.running_failure { + color: #FFFFFF; + background-color: #fffc6c; + border-color: #ff0000; +} + +.start { + border-bottom-left-radius: 10px; + -webkit-border-bottom-left-radius: 10px; + -moz-border-radius-bottomleft: 10px; + border-bottom-right-radius: 10px; + -webkit-border-bottom-right-radius: 10px; + -moz-border-radius-bottomright: 10px; +} + +.notstarted { + border-width:1px; + border-style:solid; + border-color:#aaa; +} + +.closed { + background-color: #ff0000; +} + +.closed .large { + font-size: 1.5em; + font-weight: bolder; +} + +td.Project a:hover, td.start a:hover { + color: #000; +} + +.mini-box { + text-align:center; + height:20px; + padding:0 2px; + line-height:0; + white-space:nowrap; +} + +.mini-box a { + border-radius: 0; + -webkit-border-radius: 0; + -moz-border-radius: 0; + display:block; + width:100%; + height:20px; + line-height:20px; + margin-top:-30px; +} + +.mini-closed { + -box-sizing:border-box; + -webkit-box-sizing:border-box; + border:4px solid red; +} + +/* grid styles */ + +table.Grid { + border-collapse: collapse; +} + +table.Grid tr td { + padding: 0.2em; + margin: 0px; + text-align: center; +} + +table.Grid tr td.title { + font-size: 90%; + border-right: 1px gray solid; + border-bottom: 1px gray solid; +} + +table.Grid tr td.sourcestamp { + font-size: 90%; +} + +table.Grid tr td.builder { + text-align: right; + font-size: 90%; +} + +table.Grid tr td.build { + border: 1px gray solid; +} diff --git a/tools/continuous_build/public_html/favicon.ico b/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/public_html/favicon.ico similarity index 100% rename from tools/continuous_build/public_html/favicon.ico rename to tools/continuous_build/build_internal/masters/master.tryserver.webrtc/public_html/favicon.ico diff --git a/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/slaves.cfg b/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/slaves.cfg new file mode 100644 index 000000000..90622a9ec --- /dev/null +++ b/tools/continuous_build/build_internal/masters/master.tryserver.webrtc/slaves.cfg @@ -0,0 +1,70 @@ +# -*- python -*- +# ex: set syntax=python: +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# See master.experimental/slaves.cfg for documentation. + + +def linux(): + return [ + { + 'master': 'TryServer', + 'os': 'linux', + 'version': 'lucid', + 'bits': '64', + 'builder': 'linux', + 'hostname': 'webrtc-cb-linux-slave-11', + }, + { + 'master': 'TryServer', + 'os': 'linux', + 'version': 'lucid', + 'bits': '64', + 'builder': 'linux_rel', + 'hostname': 'webrtc-cb-linux-slave-12', + }, + ] + +def mac(): + return [ + { + 'master': 'TryServer', + 'os': 'mac', + 'version': '10.7', + 'bits': '64', + 'builder': 'mac', + 'hostname': 'dhcp-172-28-249-242', + }, + { + 'master': 'TryServer', + 'os': 'mac', + 'version': '10.7', + 'bits': '64', + 'builder': 'mac_rel', + 'hostname': 'dhcp-172-28-249-244', + }, + ] + +def windows(): + return [ + { + 'master': 'TryServer', + 'os': 'win', + 'version': 'server2008', + 'bits': '64', + 'builder': 'win', + 'hostname': 'webrtc-win-x64', + }, + { + 'master': 'TryServer', + 'os': 'win', + 'version': 'server2008', + 'bits': '64', + 'builder': 'win_rel', + 'hostname': 'webrtc-win-x86', + }, + ] + +slaves = linux() + mac() + windows() diff --git a/tools/continuous_build/build_internal/masters/master.webrtc/Makefile b/tools/continuous_build/build_internal/masters/master.webrtc/Makefile new file mode 100644 index 000000000..36476d683 --- /dev/null +++ b/tools/continuous_build/build_internal/masters/master.webrtc/Makefile @@ -0,0 +1,5 @@ +# -*- makefile -*- + +BUILDBOT_PATH = $(BUILDBOT8_PATH) +MASTERPATH := master.webrtc +include ../master-common.mk diff --git a/tools/continuous_build/build_internal/masters/master.webrtc/buildbot b/tools/continuous_build/build_internal/masters/master.webrtc/buildbot new file mode 100644 index 000000000..cf3628dd5 --- /dev/null +++ b/tools/continuous_build/build_internal/masters/master.webrtc/buildbot @@ -0,0 +1,4 @@ +#!/usr/bin/python + +from buildbot.scripts import runner +runner.run() diff --git a/tools/continuous_build/build_internal/masters/master.webrtc/buildbot.tac b/tools/continuous_build/build_internal/masters/master.webrtc/buildbot.tac new file mode 100644 index 000000000..f4bca882a --- /dev/null +++ b/tools/continuous_build/build_internal/masters/master.webrtc/buildbot.tac @@ -0,0 +1,11 @@ +import os + +from twisted.application import service +from buildbot.master import BuildMaster + +basedir = os.path.dirname(os.path.abspath(__file__)) +configfile = r'master.cfg' + +application = service.Application('buildmaster') +BuildMaster(basedir, configfile).setServiceParent(application) + diff --git a/tools/continuous_build/master.cfg b/tools/continuous_build/build_internal/masters/master.webrtc/master.cfg similarity index 73% rename from tools/continuous_build/master.cfg rename to tools/continuous_build/build_internal/masters/master.webrtc/master.cfg index 2eba7e281..9bd4572f6 100755 --- a/tools/continuous_build/master.cfg +++ b/tools/continuous_build/build_internal/masters/master.webrtc/master.cfg @@ -9,66 +9,33 @@ __author__ = 'ivinnichenko@webrtc.org (Illya Vinnichenko)' -# This is a buildmaster config file for WebRTC project. It must be installed as -# 'master.cfg' in your buildmaster's base directory (although the filename -# can be changed with the --basedir option to 'mktap buildbot master'). - -# It has one job: define a dictionary named BuildmasterConfig. This -# dictionary has a variety of keys to control different aspects of the -# buildmaster. They are documented in docs/config.xhtml . - -# This is the dictionary that the buildmaster pays attention to. We also use -# a shorter alias to save typing. c = BuildmasterConfig = {} -####### BUILDSLAVES - -# the 'slaves' list defines the set of allowable buildslaves. Each element is -# a BuildSlave object, which is created with bot-name, bot-password. These -# correspond to values given to the buildslave's mktap invocation. - +import os from buildbot.buildslave import BuildSlave - -c['slaves'] = [BuildSlave('android', 'pass', max_builds=1), - BuildSlave('android-ndk', 'pass', max_builds=1), - BuildSlave('chromeos', 'pass', max_builds=1), - BuildSlave('linux-chrome', 'pass', max_builds=1), - BuildSlave('linux-clang', 'pass', max_builds=1), - BuildSlave('linux-slave-1', 'pass', max_builds=1), - BuildSlave('linux-slave-2', 'pass', max_builds=1), - BuildSlave('linux-slave-4', 'pass', max_builds=1), - BuildSlave('linux-slave-5', 'pass', max_builds=1), - BuildSlave('linux-slave-gcc-4.6', 'pass', max_builds=1), - BuildSlave('linux-valgrind', 'pass', max_builds=1), - BuildSlave('linux_video', 'pass', max_builds=1), - BuildSlave('mac-slave-3', 'pass', max_builds=1), - BuildSlave('mac-slave-2', 'pass', max_builds=1), - BuildSlave('win-slave-1', 'pass', max_builds=1), - BuildSlave('win-slave-2', 'pass', max_builds=1)] - -# 'slavePortnum' defines the TCP port to listen on. This must match the value -# configured into the buildslaves (with their --master option) - -c['slavePortnum'] = 9989 - -####### CHANGESOURCES - -# the 'change_source' setting tells the buildmaster how it should find out -# about source code changes. Any class which implements IChangeSource can be -# put here: there are several in buildbot/changes/*.py to choose from. - from buildbot.changes.pb import PBChangeSource from buildbot.changes.svnpoller import SVNPoller +from buildbot.process import factory +from buildbot.scheduler import Scheduler +from buildbot.status import html +from buildbot.status import mail +from buildbot.steps import shell +# These modules come from scripts, which must be in the PYTHONPATH. +from master import master_utils +from master import slaves_list +import config +from webrtc_buildbot import utils + +ActiveMaster = config.Master.WebRTC + +####### CHANGESOURCES source_code_svn_url = 'http://webrtc.googlecode.com/svn/trunk' svn_poller = SVNPoller(svnurl=source_code_svn_url, pollinterval=5*60, histmax=10, svnbin='/usr/bin/svn') c['change_source'] = svn_poller ####### SCHEDULERS - -## configure the Schedulers - from buildbot.scheduler import Scheduler webrtc_scheduler = Scheduler(name='all', branch=None, treeStableTimer=5*60, builderNames=['Win32Debug', @@ -161,13 +128,7 @@ windows_physical_machine_tests = utils.GetEnabledTests(PHYSICAL_MACHINE_TESTS, 'Windows') ####### FACTORIES -# Factories defines how the build is run and can be used in multiple instances -# on multiple machines, depending on how many builders are defined. - -from buildbot.process import factory -from buildbot.steps import shell - -############# Linux factories ####################################### +# Linux linux_factory_64_dbg = utils.WebRTCLinuxFactory( utils.BuildStatusOracle('linux_factory_64_dbg')) linux_factory_64_dbg.EnableBuild() @@ -227,7 +188,7 @@ android_ndk_factory = utils.WebRTCAndroidNDKFactory( utils.BuildStatusOracle('android_ndk_factory')) android_ndk_factory.EnableBuild() -############## Mac factories ####################################### +# Mac mac_factory_32_dbg = utils.WebRTCMacFactory( utils.BuildStatusOracle('mac_factory_32_dbg')) mac_factory_32_dbg.EnableBuild(build_type='both') @@ -238,7 +199,7 @@ mac_factory_32_release = utils.WebRTCMacFactory( mac_factory_32_release.EnableBuild(build_type='both', release=True) mac_factory_32_release.EnableTests(mac_normal_tests) -############# Windows factories ####################################### +# Windows win_factory_32_Debug = utils.WebRTCWinFactory( utils.BuildStatusOracle('win_factory_32_debug')) win_factory_32_Debug.EnableBuild(configuration='Debug') @@ -250,96 +211,87 @@ win_factory_32_Release.EnableBuild(configuration='Release') win_factory_32_Release.EnableTests(windows_normal_tests) ####### BUILDERS - -# the 'builders' list defines the Builders. Each one is configured with a -# dictionary, using the following keys: -# name (required): the name used to describe this builder -# slavename (required): which slave to use (must appear in c['bots']) -# builddir (required): which subdirectory to run the builder in -# factory (required): a BuildFactory to define how the build is run -# periodicBuildTime (optional): if set, force a build every N seconds - linux_builder_64_debug = { 'name': 'Linux64DBG', - 'slavename': 'linux-slave-1', + 'slavename': 'webrtc-cb-linux-slave-1', 'builddir': 'linux-slave-1', 'factory': linux_factory_64_dbg, } linux_builder_32_release = { 'name': 'Linux32Release', - 'slavename': 'linux-slave-2', + 'slavename': 'webrtc-cb-linux-slave-2', 'builddir': 'linux-slave-2', 'factory': linux_factory_32_release, } linux_builder_64_release = { 'name': 'Linux64Release', - 'slavename': 'linux-slave-5', + 'slavename': 'webrtc-cb-linux-slave-5', 'builddir': 'linux-slave-5', 'factory': linux_factory_64_release, } linux_builder_32_debug = { 'name': 'Linux32DBG', - 'slavename': 'linux-slave-4', + 'slavename': 'webrtc-cb-linux-slave-4', 'builddir': 'linux-slave-4', 'factory': linux_factory_32_dbg, } mac_builder_32_debug = { 'name': 'MacOS32DBG', - 'slavename': 'mac-slave-3', + 'slavename': 'dhcp-172-28-249-243', 'builddir': 'mac-slave-3', 'factory': mac_factory_32_dbg, } mac_builder_32_release = { 'name': 'MacOS32Release', - 'slavename': 'mac-slave-2', + 'slavename': 'dhcp-172-28-249-241', 'builddir': 'mac-slave-2', 'factory': mac_factory_32_release, } chromeos_builder = { 'name': 'ChromeOS', - 'slavename': 'chromeos', + 'slavename': 'webrtc-cb-linux-slave-3', 'builddir': 'chromeos', 'factory': chromeos_factory, } win_builder_32_debug = { 'name': 'Win32Debug', - 'slavename': 'win-slave-1', + 'slavename': 'webrtc-win2k8-1', 'builddir': 'win-32-dbg', 'factory': win_factory_32_Debug, } win_builder_32_release = { 'name': 'Win32Release', - 'slavename': 'win-slave-2', + 'slavename': 'webrtc-win2k8-2', 'builddir': 'win-32-release', 'factory': win_factory_32_Release, } linux_builder_video = { 'name': 'LinuxVideoTest', - 'slavename': 'linux_video', + 'slavename': 'webrtc-build-bot-se', 'builddir': 'video', 'factory': linux_factory_video, } linux_builder_chrome = { - 'name': "Chrome", - 'slavename': "linux-chrome", - 'builddir': "linux-chrome", + 'name': 'Chrome', + 'slavename': 'webrtc-chrome', + 'builddir': 'linux-chrome', 'factory': linux_chrome_factory, } linux_builder_clang = { 'name': 'LinuxClang', - 'slavename': 'linux-clang', + 'slavename': 'webrtc-cb-linux-slave-8', 'builddir': 'linux-clang', 'factory': linux_clang, } linux_builder_valgrind = { 'name': 'LinuxValgrind', - 'slavename': 'linux-valgrind', + 'slavename': 'webrtc-cb-linux-slave-6', 'builddir': 'linux-valgrind', 'factory': linux_valgrind, } android_builder_1 = { 'name': 'Android', - 'slavename': 'android', + 'slavename': 'webrtc-cb-linux-slave-7', 'builddir': 'android', 'factory': android_factory, } @@ -351,7 +303,7 @@ android_builder_ndk = { } linux_builder_gcc_4_6 = { 'name': 'Linux64DBG-GCC4.6', - 'slavename': 'linux-slave-gcc-4.6', + 'slavename': 'webrtc-cb-linux-slave-9', 'builddir': 'linux-slave-gcc-4.6', 'factory': linux_factory_64_dbg, } @@ -374,16 +326,25 @@ c['builders'] = [ linux_builder_chrome, ] + +####### BUILDSLAVES + +# The 'slaves' list defines the set of allowable buildslaves. List all the +# slaves registered to a builder. Remove dupes. +c['slaves'] = master_utils.AutoSetupSlaves(c['builders'], + config.Master.GetBotPassword()) + +# Slaves are loaded from slaves.cfg. +slaves = slaves_list.SlavesList('slaves.cfg', 'WebRTC') + +# Make sure everything works together. +master_utils.VerifySetup(c, slaves) + +# Adds common status and tools to this master. +master_utils.AutoSetupMaster(c, ActiveMaster, + enable_http_status_push=ActiveMaster.is_production_host) + ####### STATUS TARGETS - -# 'status' is a list of Status Targets. The results of each build will be -# pushed to these targets. buildbot/status/*.py has a variety to choose from, -# including web pages, email senders, and IRC bots. - -import os -from buildbot.status import html -from buildbot.status import mail - web_page = html.WebStatus(http_port=8010, allowForce=True) c['status'] = [web_page] @@ -396,6 +357,13 @@ if not os.getenv('BUILDBOT_DEVELOPMENT_MODE'): mode='failing') c['status'] += [email_status] +####### DB URL + +# This specifies what database buildbot uses to store change and scheduler +# state. You can leave this at its default for all but the largest +# installations. +c['db_url'] = 'sqlite:///state.sqlite' + ####### DEBUGGING OPTIONS # if you set 'debugPassword', then you can connect to the buildmaster with @@ -414,22 +382,3 @@ if not os.getenv('BUILDBOT_DEVELOPMENT_MODE'): #from buildbot import manhole #c['manhole'] = manhole.PasswordManhole('tcp:9999:interface=127.0.0.1', # 'admin', 'password') - - -####### PROJECT IDENTITY - -# the 'projectName' string will be used to describe the project that this -# buildbot is working on. For example, it is used as the title of the -# waterfall HTML page. The 'projectURL' string will be used to provide a link -# from buildbot HTML pages to your project's home page. - -c['projectName'] = 'WebRTC' -c['projectURL'] = 'http://www.webrtc.org' - -# the 'buildbotURL' string should point to the location where the buildbot's -# internal web server (usually the html.Waterfall page) is visible. This -# typically uses the port number set in the Waterfall 'status' entry, but -# with an externally-visible host name which the buildbot cannot figure out -# without some help. - -c['buildbotURL'] = 'http://webrtc-cb-linux-master.cbf.corp.google.com:8010/' diff --git a/tools/continuous_build/build_internal/masters/master.webrtc/public_html/buildbot.css b/tools/continuous_build/build_internal/masters/master.webrtc/public_html/buildbot.css new file mode 100644 index 000000000..3a136f927 --- /dev/null +++ b/tools/continuous_build/build_internal/masters/master.webrtc/public_html/buildbot.css @@ -0,0 +1,486 @@ +body { + margin-bottom:50px; +} + +body, td { + font-family: Verdana, Cursor; + font-size: 10px; + font-weight: bold; +} + +a:link,a:visited,a:active { + color: #444; +} + +a:hover { + color: #000000; +} + +table { + border-spacing: 1px 1px; +} + +table td { + padding: 3px 0px 3px 0px; + text-align: center; +} + +.closerbox { + border: 1px solid #aaa; + background-color: #eef; + border-radius: 8px; + -webkit-border-radius: 8px; + -moz-border-radius: 8px; +} + +.closerbox table { + padding: 0 4px; +} + +.closerbox .title { + background-color: #ccc; + border-top-left-radius: 8px; + -webkit-border-top-left-radius: 8px; + -moz-border-top-left-radius: 8px; + border-top-right-radius: 8px; + -webkit-border-top-right-radius: 8px; + -moz-border-top-right-radius: 8px; + padding: 0 0 2px; +} + +.Project { + width: 100px; +} + +.LastBuild, .Activity { + padding: 0 0 0 4px; +} + +.LastBuild, .Activity, .Builder, .BuildStep { + width: 155px; + max-width: 155px; +} + +/* Chromium Specific styles */ +div.BuildResultInfo { + color: #444; +} + +div.Announcement { + margin-bottom: 1em; +} + +div.Announcement > a:hover { + color: black; +} + +div.Announcement > div.Notice { + background-color: #afdaff; + padding: 0.5em; + font-size: 16px; + text-align: center; +} + +div.Announcement > div.Open { + border: 3px solid #8fdf5f; + padding: 0.5em; + font-size: 16px; + text-align: center; +} + +div.Announcement > div.Closed { + border: 5px solid #e98080; + padding: 0.5em; + font-size: 24px; + font-weight: bold; + text-align: center; +} + +td.Time { + color: #000; + border-bottom: 1px solid #aaa; + background-color: #eee; +} + +td.Activity, td.Change, td.Builder { + color: #333333; + background-color: #CCCCCC; +} + +td.Change { + border-radius: 5px; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; +} +td.Event { + color: #777; + background-color: #ddd; + border-radius: 5px; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; +} + +td.Activity { + border-top-left-radius: 10px; + -webkit-border-top-left-radius: 10px; + -moz-border-radius-topleft: 10px; + min-height: 20px; + padding: 2px 0 2px 0; +} + +td.idle, td.waiting, td.offline, td.building { + border-top-left-radius: 0px; + -webkit-border-top-left-radius: 0px; + -moz-border-radius-topleft: 0px; +} + +.LastBuild { + border-top-left-radius: 5px; + -webkit-border-top-left-radius: 5px; + -moz-border-radius-topleft: 5px; + border-top-right-radius: 5px; + -webkit-border-top-right-radius: 5px; + -moz-border-radius-topright: 5px; +} + +/* Console view styles */ + +td.DevRev { + padding: 4px 8px 4px 8px; + color: #333333; + border-top-left-radius: 5px; + -webkit-border-top-left-radius: 5px; + -moz-border-radius-topleft: 5px; + background-color: #eee; + width: 1%; +} + +td.DevRevCollapse { + border-bottom-left-radius: 5px; + -webkit-border-bottom-left-radius: 5px; + -moz-border-radius-bottomleft: 5px; +} + +td.DevName { + padding: 4px 8px 4px 8px; + color: #333333; + background-color: #eee; + width: 1%; + text-align: left; +} + +td.DevStatus { + padding: 4px 4px 4px 4px; + color: #333333; + background-color: #eee; +} + +td.DevSlave { + padding: 4px 4px 4px 4px; + color: #333333; + background-color: #eee; +} + +td.first { + border-top-left-radius: 5px; + -webkit-border-top-left-radius: 5px; + -moz-border-radius-topleft: 5px; +} + +td.last { + border-top-right-radius: 5px; + -webkit-border-top-right-radius: 5px; + -moz-border-radius-topright: 5px; +} + +td.DevStatusCategory { + border-radius: 5px; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; + border-width:1px; + border-style:solid; +} + +td.DevStatusCollapse { + border-bottom-right-radius: 5px; + -webkit-border-bottom-right-radius: 5px; + -moz-border-radius-bottomright: 5px; +} + +td.DevDetails { + font-weight: normal; + padding: 8px 8px 8px 8px; + color: #333333; + background-color: #eee; + text-align: left; +} + +td.DevComment { + font-weight: normal; + padding: 8px 8px 8px 8px; + color: #333333; + border-bottom-right-radius: 5px; + -webkit-border-bottom-right-radius: 5px; + -moz-border-radius-bottomright: 5px; + border-bottom-left-radius: 5px; + -webkit-border-bottom-left-radius: 5px; + -moz-border-radius-bottomleft: 5px; + background-color: #eee; + text-align: left; +} + +td.Alt { + background-color: #CCCCCC; +} + +.legend { + border-radius: 5px; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; + width: 100px; + max-width: 100px; + text-align:center; + padding: 2px 2px 2px 2px; + height:14px; + white-space:nowrap; +} + +.DevStatusBox { + text-align:center; + height:20px; + padding:0 2px; + line-height:0; + white-space:nowrap; +} + +.DevStatusBox a { + opacity: 0.85; + border-width:1px; + border-style:solid; + border-radius: 4px; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + display:block; + width:90%; + height:20px; + line-height:20px; + margin-left: auto; + margin-right: auto; +} + +.DevSlaveBox { + text-align:center; + height:10px; + padding:0 2px; + line-height:0; + white-space:nowrap; +} + +.DevSlaveBox a { + opacity: 0.85; + border-width:1px; + border-style:solid; + border-radius: 4px; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + display:block; + width:90%; + height:10px; + line-height:20px; + margin-left: auto; + margin-right: auto; +} + +a.noround { + border-radius: 0px; + -webkit-border-radius: 0px; + -moz-border-radius: 0px; + position: relative; + margin-top: -8px; + margin-bottom: -8px; + height: 36px; + border-top-width: 0; + border-bottom-width: 0; +} + +a.begin { + border-top-width:1px; + position: relative; + margin-top: 0px; + margin-bottom: -7px; + height: 27px; + border-top-left-radius: 4px; + -webkit-border-top-left-radius: 4px; + -moz-border-radius-topleft: 4px; + border-top-right-radius: 4px; + -webkit-border-top-right-radius: 4px; + -moz-border-radius-topright: 4px; +} + +a.end { + border-bottom-width:1px; + position: relative; + margin-top: -7px; + margin-bottom: 0px; + height: 27px; + border-bottom-left-radius: 4px; + -webkit-border-bottom-left-radius: 4px; + -moz-border-radius-bottomleft: 4px; + border-bottom-right-radius: 4px; + -webkit-border-bottom-right-radius: 4px; + -moz-border-radius-bottomright: 4px; +} + +.center_align { + text-align: center; +} + +.right_align { + text-align: right; +} + +.left_align { + text-align: left; +} + +div.BuildWaterfall { + border-radius: 7px; + -webkit-border-radius: 7px; + -moz-border-radius: 7px; + position: absolute; + left: 0px; + top: 0px; + background-color: #FFFFFF; + padding: 4px 4px 4px 4px; + float: left; + display: none; + border-width: 1px; + border-style: solid; +} + +/* LastBuild, BuildStep states */ +.success { + color: #FFFFFF; + background-color: #8fdf5f; + border-color: #4F8530; +} + +.failure { + color: #FFFFFF; + background-color: #e98080; + border-color: #A77272; +} + +.warnings { + color: #FFFFFF; + background-color: #ffc343; + border-color: #C29D46; +} + +.never { + color: #FFFFFF; + background-color: #f0f0e0; + border-color: #A77272; +} + +.exception, td.offline { + color: #FFFFFF; + background-color: #e0b0ff; + border-color: #ACA0B3; +} + +.start,.running, td.building { + color: #666666; + background-color: #fffc6c; + border-color: #C5C56D; +} + +.running_failure { + color: #FFFFFF; + background-color: #fffc6c; + border-color: #ff0000; +} + +.start { + border-bottom-left-radius: 10px; + -webkit-border-bottom-left-radius: 10px; + -moz-border-radius-bottomleft: 10px; + border-bottom-right-radius: 10px; + -webkit-border-bottom-right-radius: 10px; + -moz-border-radius-bottomright: 10px; +} + +.notstarted { + border-width:1px; + border-style:solid; + border-color:#aaa; +} + +.closed { + background-color: #ff0000; +} + +.closed .large { + font-size: 1.5em; + font-weight: bolder; +} + +td.Project a:hover, td.start a:hover { + color: #000; +} + +.mini-box { + text-align:center; + height:20px; + padding:0 2px; + line-height:0; + white-space:nowrap; +} + +.mini-box a { + border-radius: 0; + -webkit-border-radius: 0; + -moz-border-radius: 0; + display:block; + width:100%; + height:20px; + line-height:20px; + margin-top:-30px; +} + +.mini-closed { + -box-sizing:border-box; + -webkit-box-sizing:border-box; + border:4px solid red; +} + +/* grid styles */ + +table.Grid { + border-collapse: collapse; +} + +table.Grid tr td { + padding: 0.2em; + margin: 0px; + text-align: center; +} + +table.Grid tr td.title { + font-size: 90%; + border-right: 1px gray solid; + border-bottom: 1px gray solid; +} + +table.Grid tr td.sourcestamp { + font-size: 90%; +} + +table.Grid tr td.builder { + text-align: right; + font-size: 90%; +} + +table.Grid tr td.build { + border: 1px gray solid; +} diff --git a/tools/continuous_build/build_internal/masters/master.webrtc/public_html/favicon.ico b/tools/continuous_build/build_internal/masters/master.webrtc/public_html/favicon.ico new file mode 100644 index 000000000..324a6caf6 Binary files /dev/null and b/tools/continuous_build/build_internal/masters/master.webrtc/public_html/favicon.ico differ diff --git a/tools/continuous_build/build_internal/masters/master.webrtc/slaves.cfg b/tools/continuous_build/build_internal/masters/master.webrtc/slaves.cfg new file mode 100644 index 000000000..d8bbefbe2 --- /dev/null +++ b/tools/continuous_build/build_internal/masters/master.webrtc/slaves.cfg @@ -0,0 +1,150 @@ +# -*- python -*- +# ex: set syntax=python: +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# See master.experimental/slaves.cfg for documentation. + + +def linux(): + return [ + { + 'master': 'WebRTC', + 'os': 'linux', + 'version': 'lucid', + 'bits': '64', + 'builder': 'Linux64DBG', + 'hostname': 'webrtc-cb-linux-slave-1', + }, + { + 'master': 'WebRTC', + 'os': 'linux', + 'version': 'lucid', + 'bits': '64', + 'builder': 'Linux32Release', + 'hostname': 'webrtc-cb-linux-slave-2', + }, + { + 'master': 'WebRTC', + 'os': 'linux', + 'version': 'lucid', + 'bits': '64', + 'builder': 'ChromeOS', + 'hostname': 'webrtc-cb-linux-slave-3', + }, + { + 'master': 'WebRTC', + 'os': 'linux', + 'version': 'lucid', + 'bits': '64', + 'builder': 'Linux32DBG', + 'hostname': 'webrtc-cb-linux-slave-4', + }, + { + 'master': 'WebRTC', + 'os': 'linux', + 'version': 'lucid', + 'bits': '64', + 'builder': 'Linux64Release', + 'hostname': 'webrtc-cb-linux-slave-5', + }, + { + 'master': 'WebRTC', + 'os': 'linux', + 'version': 'lucid', + 'bits': '64', + 'builder': 'LinuxValgrind', + 'hostname': 'webrtc-cb-linux-slave-6', + }, + { + 'master': 'WebRTC', + 'os': 'linux', + 'version': 'lucid', + 'bits': '64', + 'builder': 'Android', + 'hostname': 'webrtc-cb-linux-slave-7', + }, + { + 'master': 'WebRTC', + 'os': 'linux', + 'version': 'lucid', + 'bits': '64', + 'builder': 'LinuxClang', + 'hostname': 'webrtc-cb-linux-slave-8', + }, + { + 'master': 'WebRTC', + 'os': 'linux', + 'version': 'lucid', + 'bits': '64', + 'builder': 'Linux64DBG-GCC4.6', + 'hostname': 'webrtc-cb-linux-slave-9', + }, + { + 'master': 'WebRTC', + 'os': 'linux', + 'version': 'lucid', + 'bits': '64', + 'builder': 'AndroidNDK', + 'hostname': 'webrtc-cb-linux-slave-10', + }, + { + 'master': 'WebRTC', + 'os': 'linux', + 'version': 'lucid', + 'bits': '64', + 'builder': 'LinuxVideoTest', + 'hostname': 'webrtc-build-bot-se', + }, + { + 'master': 'WebRTC', + 'os': 'linux', + 'version': 'lucid', + 'bits': '64', + 'builder': 'Chrome', + 'hostname': 'webrtc-chrome', + }, + ] + +def mac(): + return [ + { + 'master': 'WebRTC', + 'os': 'mac', + 'version': '10.7', + 'bits': '64', + 'builder': 'MacOS32DBG', + 'hostname': 'dhcp-172-28-249-243', + }, + { + 'master': 'WebRTC', + 'os': 'mac', + 'version': '10.7', + 'bits': '64', + 'builder': 'MacOS32Release', + 'hostname': 'dhcp-172-28-249-241', + }, + ] + +def windows(): + return [ + { + 'master': 'WebRTC', + 'os': 'win', + 'version': 'server2008', + 'bits': '64', + 'builder': 'Win32Debug', + 'hostname': 'webrtc-win2k8-1', + }, + { + 'master': 'WebRTC', + 'os': 'win', + 'version': 'server2008', + 'bits': '64', + 'builder': 'Win32Release', + 'hostname': 'webrtc-win2k8-2', + }, + ] + +slaves = linux() + mac() + windows() diff --git a/tools/continuous_build/clean_old_files.py b/tools/continuous_build/build_internal/scripts/clean_old_files.py similarity index 100% rename from tools/continuous_build/clean_old_files.py rename to tools/continuous_build/build_internal/scripts/clean_old_files.py diff --git a/tools/continuous_build/clean_third_party_gcda.sh b/tools/continuous_build/build_internal/scripts/clean_third_party_gcda.sh similarity index 100% rename from tools/continuous_build/clean_third_party_gcda.sh rename to tools/continuous_build/build_internal/scripts/clean_third_party_gcda.sh diff --git a/tools/continuous_build/generate_coverage_html.sh b/tools/continuous_build/build_internal/scripts/generate_coverage_html.sh similarity index 100% rename from tools/continuous_build/generate_coverage_html.sh rename to tools/continuous_build/build_internal/scripts/generate_coverage_html.sh diff --git a/tools/continuous_build/build_internal/scripts/webrtc_buildbot/__init__.py b/tools/continuous_build/build_internal/scripts/webrtc_buildbot/__init__.py new file mode 100755 index 000000000..e6f0468af --- /dev/null +++ b/tools/continuous_build/build_internal/scripts/webrtc_buildbot/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. diff --git a/tools/continuous_build/webrtc_buildbot/utils.py b/tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py similarity index 87% rename from tools/continuous_build/webrtc_buildbot/utils.py rename to tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py index ab819f2a9..6bb423fa9 100755 --- a/tools/continuous_build/webrtc_buildbot/utils.py +++ b/tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py @@ -18,6 +18,9 @@ from buildbot.process import properties from buildbot.process.properties import WithProperties from buildbot.steps.shell import ShellCommand +from master import chromium_step +from master.factory import gclient_factory + # Defines the order of the booleans of the supported platforms in the test # dictionaries in master.cfg. SUPPORTED_PLATFORMS = ('Linux', 'Mac', 'Windows') @@ -71,16 +74,19 @@ class WebRTCFactory(factory.BuildFactory): can be overridden to create customized build sequences. """ - def __init__(self, build_status_oracle): + def __init__(self, build_status_oracle, is_try_slave=False): """Creates the abstract factory. Args: build_status_oracle: An instance of BuildStatusOracle which is used to keep track of our build state. + is_try_slave: If this bot is a try slave. Needed since we're handling + some things differently between normal slaves and try slaves. """ factory.BuildFactory.__init__(self) self.build_status_oracle = build_status_oracle + self.is_try_slave = is_try_slave self.properties = properties.Properties() self.gyp_params = [] self.release = False @@ -106,8 +112,7 @@ class WebRTCFactory(factory.BuildFactory): self.EnableTest(test) def AddCommonStep(self, cmd, descriptor='', workdir=WEBRTC_TRUNK_DIR, - number_of_retries=0, halt_build_on_failure=True, - warn_on_failure=False): + halt_build_on_failure=True, warn_on_failure=False): """Adds a step which will run as a shell command on the slave. NOTE: you are recommended to use this method to add new shell commands @@ -127,7 +132,6 @@ class WebRTCFactory(factory.BuildFactory): responsible for making sure that the slashes are flipped the right way depending on platform, which means you can't use the default value if the step will run on a Windows machine. - number_of_retries: Number of times to retry the command, if it fails. halt_build_on_failure: Stops the build dead in its tracks if this step fails. Use for critical steps. This option does not make sense with warn_on_failure. @@ -142,9 +146,8 @@ class WebRTCFactory(factory.BuildFactory): # Add spaces to wrap long test names to make waterfall output more compact. wrapped_text = self._WrapLongLines(descriptor) - self.addStep(MonitoredRetryingShellCommand( + self.addStep(MonitoredShellCommand( build_status_oracle=self.build_status_oracle, - number_of_retries=number_of_retries, command=cmd, workdir=workdir, description=wrapped_text + ['running...'], @@ -160,7 +163,7 @@ class WebRTCFactory(factory.BuildFactory): Smart clean only cleans the whole repository if the build status oracle thinks the last build failed. Otherwise it cleans just the build output. """ - self.addStep(SmartClean(self.build_status_oracle, + self.addStep(SmartClean(self.build_status_oracle, self.is_try_slave, workdir=WEBRTC_BUILD_DIR)) def AddCommonTestRunStep(self, test, descriptor='', cmd=None): @@ -206,22 +209,27 @@ class WebRTCFactory(factory.BuildFactory): workdir=WEBRTC_BUILD_DIR, descriptor='gclient_config') - def AddGclientSyncStep(self, force_sync): - """Helper method for invoking gclient sync. + def AddGclientSyncStep(self): + """Helper method for invoking gclient sync.""" + gclient_spec = self._ConfigureWhatToBuild() + env = self._GetEnvironmentWithDisabledDepotToolsUpdate() - Will retry if the operation fails. Runs in the bot's build directory - (e.g. one step above trunk). - - Args: - force_sync: If the sync should be forced, i.e. update even for - unchanged modules (known to be required for Windows sometimes). - """ - cmd = ['gclient', 'sync'] - if force_sync: - cmd.append('--force') - self.AddCommonStep(cmd, descriptor='Sync', - workdir=WEBRTC_BUILD_DIR, - number_of_retries=GCLIENT_RETRIES) + # Define how the GClient command shall be executed. + # Try 4+1=5 times, 10 seconds apart. + retry = (10, 4) + # Subversion timeout is by default 2 minutes; we allow 5 minutes. + timeout = 60*5 + # Removal can take a long time. Allow 15 minutes. + rm_timeout = 60*15 + self.addStep(chromium_step.GClient, + gclient_spec=gclient_spec, + svnurl=WEBRTC_SVN_LOCATION, + workdir='build', + mode='update', + env=env, + retry=retry, + timeout=timeout, + rm_timeout=rm_timeout) def AddCommonGYPStep(self, gyp_file, gyp_params=[], descriptor='gyp'): """Helper method for invoking GYP on WebRTC. @@ -242,6 +250,18 @@ class WebRTCFactory(factory.BuildFactory): cmd += gyp_params + self.gyp_params self.AddCommonStep(cmd=cmd, descriptor=descriptor) + def _ConfigureWhatToBuild(self): + """Returns a string with the contents of a .gclient file.""" + solution = gclient_factory.GClientSolution(name='trunk', + svn_url=WEBRTC_SVN_LOCATION) + return 'solutions = [ %s ]' % solution.GetSpec() + + def _GetEnvironmentWithDisabledDepotToolsUpdate(self): + """Returns a dictionary of environment variables to be used by GClient.""" + env = {} + env['DEPOT_TOOLS_UPDATE'] = '0' + return env + def _WrapLongLines(self, string_list, max_line_length=25, wrap_character='_'): """ Creates a list with wrapped strings for lines that are too long. @@ -307,52 +327,45 @@ class BuildStatusOracle: return os.path.join(self.master_work_dir, self.builder_name + ".failed") -class MonitoredRetryingShellCommand(ShellCommand): - """Wraps a shell command and notifies the oracle if the command fails. - - If the command fails, there's an option to retry it a number of times. - Default behavior is to not retry.""" - - def __init__(self, build_status_oracle, number_of_retries=0, **kwargs): +class MonitoredShellCommand(ShellCommand): + """Wraps a shell command and notifies the oracle if the command fails.""" + def __init__(self, build_status_oracle, **kwargs): ShellCommand.__init__(self, **kwargs) - self.addFactoryArguments(build_status_oracle=build_status_oracle, - number_of_retries=number_of_retries) + self.addFactoryArguments(build_status_oracle=build_status_oracle) self.build_status_oracle = build_status_oracle - self.number_of_retries = number_of_retries def finished(self, results): if (results == buildbot.status.builder.FAILURE or results == buildbot.status.builder.EXCEPTION): - if self.number_of_retries > 0: - self.number_of_retries -= 1 - self.start() - return - else: - self.build_status_oracle.SetLastBuildAsFailed() + self.build_status_oracle.SetLastBuildAsFailed() ShellCommand.finished(self, results) class SmartClean(ShellCommand): """Cleans the repository fully or partially depending on the build state.""" - def __init__(self, build_status_oracle, **kwargs): + def __init__(self, build_status_oracle, is_try_slave, **kwargs): ShellCommand.__init__(self, **kwargs) - self.addFactoryArguments(build_status_oracle=build_status_oracle) + self.addFactoryArguments(build_status_oracle=build_status_oracle, + is_try_slave=is_try_slave) self.haltOnFailure = True self.build_status_oracle = build_status_oracle + self.is_try_slave = is_try_slave def start(self): - if self.build_status_oracle.LastBuildFailed(): - self.build_status_oracle.ForgetLastBuild() - self.description = ['Nuke Repository', '(Previous Failed)'] - self.setCommand(['rm', '-rf', 'trunk']) - else: + # Always do normal clean for try slaves, since nuking confuses the Chromium + # scripts' GClient sync step. + if self.is_try_slave or self.build_status_oracle.LastBuildFailed(): self.description = ['Clean'] self.setCommand('rm -rf trunk/out && ' 'rm -rf trunk/xcodebuild &&' 'rm -rf trunk/build/Debug &&' 'rm -rf trunk/build/Release') + else: + self.build_status_oracle.ForgetLastBuild() + self.description = ['Nuke Repository', '(Previous Failed)'] + self.setCommand(['rm', '-rf', 'trunk']) ShellCommand.start(self) @@ -454,7 +467,7 @@ class WebRTCChromeFactory(WebRTCFactory): def EnableBuild(self): self.AddCommonStep(['rm', '-rf', 'src'], workdir=WEBRTC_BUILD_DIR, descriptor='Cleanup') - self.AddGclientSyncStep(force_sync=True) + self.AddGclientSyncStep() self.AddCommonMakeStep('chrome') def AddCommonMakeStep(self, target, make_extra=None): @@ -472,8 +485,9 @@ class WebRTCLinuxFactory(WebRTCFactory): This factory is quite configurable and can run a variety of builds. """ - def __init__(self, build_status_oracle, valgrind_enabled=False): - WebRTCFactory.__init__(self, build_status_oracle) + def __init__(self, build_status_oracle, is_try_slave=False, + valgrind_enabled=False): + WebRTCFactory.__init__(self, build_status_oracle, is_try_slave) self.build_enabled = False self.coverage_enabled = False @@ -514,7 +528,7 @@ class WebRTCLinuxFactory(WebRTCFactory): self.gyp_params.append('-D' + gyp_define) else: self.AddGclientConfigStep() - self.AddGclientSyncStep(force_sync=False) + self.AddGclientSyncStep() if chrome_os: self.gyp_params.append('-Dchromeos=1') @@ -666,8 +680,8 @@ class WebRTCLinuxFactory(WebRTCFactory): class WebRTCMacFactory(WebRTCFactory): """Sets up the Mac build, both for make and xcode.""" - def __init__(self, build_status_oracle): - WebRTCFactory.__init__(self, build_status_oracle) + def __init__(self, build_status_oracle, is_try_slave=False): + WebRTCFactory.__init__(self, build_status_oracle, is_try_slave) self.build_type = 'both' self.allowed_build_types = ['both', 'xcode', 'make'] @@ -681,7 +695,7 @@ class WebRTCMacFactory(WebRTCFactory): self.build_type = build_type self.AddSmartCleanStep() self.AddGclientConfigStep() - self.AddGclientSyncStep(force_sync=True) + self.AddGclientSyncStep() if self.build_type == 'make' or self.build_type == 'both': self.AddCommonGYPStep('webrtc.gyp', gyp_params=['-f', 'make'], @@ -725,20 +739,22 @@ class WebRTCWinFactory(WebRTCFactory): Allows building with Debug, Release or both in sequence. """ - def __init__(self, build_status_oracle): - WebRTCFactory.__init__(self, build_status_oracle) + # Must provide full path to the command since we cannot add custom paths to + # the PATH environment variable when using Chromium buildbot startup scripts. + BUILD_CMD = r'C:\Windows\Microsoft.NET\Framework\v3.5\msbuild.exe' + + def __init__(self, build_status_oracle, is_try_slave=False): + WebRTCFactory.__init__(self, build_status_oracle, is_try_slave) self.configuration = 'Debug' self.platform = 'x64' self.allowed_platforms = ['x64', 'Win32'] self.allowed_configurations = ['Debug', 'Release', 'both'] def AddCommonStep(self, cmd, descriptor='', workdir=WEBRTC_TRUNK_DIR, - number_of_retries=0, halt_build_on_failure=True, - warn_on_failure=False): + halt_build_on_failure=True, warn_on_failure=False): workdir = workdir.replace('/', '\\') WebRTCFactory.AddCommonStep(self, cmd, descriptor, workdir, - number_of_retries, halt_build_on_failure, - warn_on_failure) + halt_build_on_failure, warn_on_failure) def EnableBuild(self, platform='Win32', configuration='Debug'): if platform not in self.allowed_platforms: @@ -761,23 +777,40 @@ class WebRTCWinFactory(WebRTCFactory): cmd = '%WINDIR%\\system32\\taskkill /f /im svn.exe || set ERRORLEVEL=0' self.AddCommonStep(cmd, 'svnkill') + # TODO(kjellander): Enable for normal slaves too when all are moved over to + # the new slave architecture. + if self.is_try_slave: + # Run the Chromium kill process script. It requires the handle.exe to be + # copied into third_party/psutils in order to not fail. + # Download from: + # http://technet.microsoft.com/en-us/sysinternals/bb896655.aspx + # To avoid having to modify kill_processes.py, we set the working dir to + # the build dir (three levels up from the build dir that contains + # third_party/psutils). + cmd = 'python ..\\..\\..\\scripts\\slave\\kill_processes.py' + self.AddCommonStep(cmd, 'taskkill', workdir=WEBRTC_BUILD_DIR) + # Now do the clean + build. self.AddSmartCleanStep() self.AddGclientConfigStep() - self.AddGclientSyncStep(force_sync=True) + self.AddGclientSyncStep() if self.configuration == 'Debug' or self.configuration == 'both': - cmd = ['msbuild', 'webrtc.sln', '/t:Clean', '/verbosity:diagnostic', + cmd = [WebRTCWinFactory.BUILD_CMD, 'webrtc.sln', '/t:Clean', + '/verbosity:diagnostic', '/p:Configuration=Debug;Platform=%s' % (self.platform)] self.AddCommonStep(cmd, descriptor='Build(Clean)') - cmd = ['msbuild', 'webrtc.sln', '/verbosity:diagnostic', + cmd = [WebRTCWinFactory.BUILD_CMD, 'webrtc.sln', + '/verbosity:diagnostic', '/p:Configuration=Debug;Platform=%s' % (self.platform)] self.AddCommonStep(cmd, descriptor='Build(Debug)') if self.configuration == 'Release' or self.configuration == 'both': - cmd = ['msbuild', 'webrtc.sln', '/t:Clean', '/verbosity:diagnostic', + cmd = [WebRTCWinFactory.BUILD_CMD, 'webrtc.sln', '/t:Clean', + '/verbosity:diagnostic', '/p:Configuration=Release;Platform=%s' % (self.platform)] self.AddCommonStep(cmd, descriptor='Build(Clean)') - cmd = ['msbuild', 'webrtc.sln', '/verbosity:diagnostic', + cmd = [WebRTCWinFactory.BUILD_CMD, 'webrtc.sln', + '/verbosity:diagnostic', '/p:Configuration=Release;Platform=%s' % (self.platform)] self.AddCommonStep(cmd, descriptor='Build(Release)') diff --git a/tools/continuous_build/webrtc_buildbot/utils_test.py b/tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils_test.py similarity index 100% rename from tools/continuous_build/webrtc_buildbot/utils_test.py rename to tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils_test.py diff --git a/tools/continuous_build/build_internal/site_config/config_private.py b/tools/continuous_build/build_internal/site_config/config_private.py new file mode 100644 index 000000000..c7d366e23 --- /dev/null +++ b/tools/continuous_build/build_internal/site_config/config_private.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +# This file is based on the Chromium config.py located at +# /trunk/tools/build/site_config/config_default.py of the Chromium tools. + +import socket + +class Master(object): + # Repository URLs used by the SVNPoller and 'gclient config'. + server_url = 'http://webrtc.googlecode.com' + git_server_url = 'http://webrtc.googlecode.com/git' + repo_root = '/svn' + + # External repos. + googlecode_url = 'http://%s.googlecode.com/svn' + sourceforge_url = 'https://%(repo)s.svn.sourceforge.net/svnroot/%(repo)s' + + # Directly fetches from anonymous webkit svn server. + webkit_root_url = 'http://svn.webkit.org/repository/webkit' + + llvm_url = 'http://llvm.org/svn/llvm-project' + + # Other non-redistributable repositories. + repo_root_internal = None + trunk_internal_url = None + trunk_internal_url_src = None + gears_url_internal = None + o3d_url_internal = None + nacl_trunk_url_internal = None + nacl_url_internal = None + + syzygy_internal_url = None + + # Other non-redistributable repositories. + repo_root_internal = None + trunk_internal_url = None + trunk_internal_url_src = None + + # Please change this accordingly. + master_domain = 'webrtc.org' + permitted_domains = ('webrtc.org',) + + # Your smtp server to enable mail notifications. + smtp = 'smtp' + + # By default, bot_password will be filled in by config.GetBotPassword(); + # if the private config wants to override this, it can do so. + bot_password = None + + class _Base(object): + # Master address. You should probably copy this file in another svn repo + # so you can override this value on both the slaves and the master. + master_host = 'webrtc-cb-linux-master.cbf.corp.google.com' + # If set to True, the master will do nasty stuff like closing the tree, + # sending emails or other similar behaviors. Don't change this value unless + # you modified the other settings extensively. + is_production_host = socket.getfqdn() == master_host + # Additional email addresses to send gatekeeper (automatic tree closage) + # notifications. Unnecessary for experimental masters and try servers. + tree_closing_notification_recipients = [] + # 'from:' field for emails sent from the server. + from_address = 'webrtc-cb-watchlist@google.com' + # Code review site to upload results. You should setup your own Rietveld + # instance with the code at + # http://code.google.com/p/rietveld/source/browse/#svn/branches/chromium + # You can host your own private rietveld instance on Django, see + # http://code.google.com/p/google-app-engine-django and + # http://code.google.com/appengine/articles/pure_django.html + code_review_site = 'https://webrtc-codereview.appspot.com/status_listener' + + # For the following values, they are used only if non-0. Do not set them + # here, set them in the actual master configuration class. + + # Used for the waterfall URL and the waterfall's WebStatus object. + master_port = 0 + # Which port slaves use to connect to the master. + slave_port = 0 + # The alternate read-only page. Optional. + master_port_alt = 0 + # HTTP port for try jobs. + try_job_port = 0 + + ## Chrome related + + class _ChromiumBase(_Base): + # Tree status urls. You should fork the code from tools/chromium-status/ and + # setup your own AppEngine instance (or use directly Django to create a + # local instance). + # Defaulting urls that are used to POST data to 'localhost' so a local dev + # server can be used for testing and to make sure nobody updates the tree + # status by error! + # + # This url is used for HttpStatusPush: + base_app_url = 'http://localhost:8080' + # HTTP url that should return 0 or 1, depending if the tree is open or + # closed. It is also used as POST to update the tree status. + tree_status_url = base_app_url + '/status' + # Used by LKGR to POST data. + store_revisions_url = base_app_url + '/revisions' + # Used by the try server to sync to the last known good revision: + last_good_url = 'http://webrtc-dashboard.appspot.com/lkgr' + + class WebRTC(_ChromiumBase): + # Used by the waterfall display. + project_name = 'WebRTC' + master_port = 9010 + slave_port = 9112 + master_port_alt = 9014 + + class WebRTCMemory(_ChromiumBase): + project_name = 'WebRTC Memory' + master_port = 9014 + slave_port = 9119 + master_port_alt = 9047 + + class WebRTCPerf(_ChromiumBase): + project_name = 'WebRTC Perf' + master_port = 9050 + slave_port = 9151 + master_port_alt = 9052 + + class TryServer(_ChromiumBase): + project_name = 'WebRTC Try Server' + master_port = 9011 + slave_port = 9113 + master_port_alt = 9015 + try_job_port = 9018 + # The svn repository to poll to grab try patches. For chrome, we use a + # separate repo to put all the diff files to be tried. + svn_url = None + +class Archive(object): + archive_host = 'localhost' + # Skip any filenames (exes, symbols, etc.) starting with these strings + # entirely, typically because they're not built for this distribution. + exes_to_skip_entirely = [] + # Web server base path. + www_dir_base = "\\\\" + archive_host + "\\www\\" + + @staticmethod + def Internal(): + pass + + +class Distributed(object): + """Not much to describe.""" diff --git a/tools/continuous_build/build_internal/symsrc/webrtc_buildbot/__init__.py b/tools/continuous_build/build_internal/symsrc/webrtc_buildbot/__init__.py new file mode 100755 index 000000000..e6f0468af --- /dev/null +++ b/tools/continuous_build/build_internal/symsrc/webrtc_buildbot/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. diff --git a/tools/continuous_build/build_internal/symsrc/webrtc_buildbot/slave_utils.py b/tools/continuous_build/build_internal/symsrc/webrtc_buildbot/slave_utils.py new file mode 100644 index 000000000..e16117c27 --- /dev/null +++ b/tools/continuous_build/build_internal/symsrc/webrtc_buildbot/slave_utils.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +__author__ = 'kjellander@webrtc.org (Henrik Kjellander)' + +"""Contains a customized GClient class for WebRTC use. + +It is essential that this class is imported after the chromium_commands so it +can overwrite its registration of the 'gclient' command. +This should by done by adding: + +from webrtc_buildbot import slave_utils + +into the buildbot.tac file of the try slave. +""" + +from slave import chromium_commands + +try: + # Buildbot 0.8.x + # Unable to import 'XX' + # pylint: disable=F0401 + from buildslave.commands.registry import commandRegistry +except ImportError: + # Buildbot 0.7.12 + # Unable to import 'XX' + # pylint: disable=E0611,E1101,F0401 + from buildbot.slave.registry import registerSlaveCommand + +class GClient(chromium_commands.GClient): + def doPatch(self, res): + # For some unknown reason, the patch tuple recieved only have two items: + # the patch level and the actual diff contents. The third optional root + # argument that tells which base directory to use for the patch is not + # included. Since we need this to be able to patch properly in WebRTC, we'll + # just create a new tuple with this added. + self.patch = (self.patch[0], self.patch[1], 'trunk') + return chromium_commands.GClient.doPatch(self, res) + + +def RegisterGclient(): + try: + # This version should work on BB 8.3 + + # We run this code in a try because it fails with an assertion if + # the module is loaded twice. + commandRegistry['gclient'] = 'webrtc_buildbot.slave_utils.GClient' + return + except (AssertionError, NameError): + pass + + try: + # This version should work on BB 7.12 + # We run this code in a try because it fails with an assertion if + # the module is loaded twice. + registerSlaveCommand('gclient', GClient, commands.command_version) + except (AssertionError, NameError): + pass + +RegisterGclient() diff --git a/tools/continuous_build/public_html/README b/tools/continuous_build/public_html/README deleted file mode 100644 index 5f5900566..000000000 --- a/tools/continuous_build/public_html/README +++ /dev/null @@ -1,18 +0,0 @@ -Modify the look of the buildbot master's waterfall page -------------------------------------------------------- - -To deploy these changes, follow these instructions: - -* Copy the favicon.ico file to master/public_html/ on the buildbot master - machine. - -* Edit the master/public_html/index.html and add the following to the - element: - - - -* Edit the master/public_html/buildbot.css and change the section with - .LastBuild, .Activity { - width: 230px; - - to be 160px instead. Save the file and exit. diff --git a/tools/continuous_build/webrtc_buildbot/__init__.py b/tools/continuous_build/webrtc_buildbot/__init__.py deleted file mode 100755 index 869bd84b3..000000000 --- a/tools/continuous_build/webrtc_buildbot/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# -*- python -*- -# ex: set syntax=python: