generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
webrtc/tools/continuous_build/build_internal/masters/master.webrtc/master.cfg
kjellander@webrtc.org 08d9928443 ThreadSanitizer (TSAN) and AddressSanitizer (ASAN) bots 
BUG=None
TEST=Tested on local master with the live slaves.

Review URL: https://webrtc-codereview.appspot.com/571011

git-svn-id: http://webrtc.googlecode.com/svn/trunk@2258 4adac7df-926f-26a2-2b94-8c16560cd09d
2012-05-18 14:33:06 +00:00

479 lines
18 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python
# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
__author__ = 'kjellander@webrtc.org (Henrik Kjellander)'
c = BuildmasterConfig = {}
import os
from buildbot.buildslave import BuildSlave
from buildbot.changes.pb import PBChangeSource
from buildbot.changes.svnpoller import SVNPoller
from buildbot.process import factory
from buildbot.scheduler import Scheduler
from buildbot.schedulers import timed
from buildbot.status import html
from buildbot.status import mail
from buildbot.status.web import grid
from buildbot.steps import shell
# These modules come from scripts, which must be in the PYTHONPATH.
from master import master_utils
from master import slaves_list
import config
from webrtc_buildbot import utils
ActiveMaster = config.Master.WebRTC
####### CHANGESOURCES
import master_source_cfg
master_source_cfg.ConfigureChangeSource(config, c)
####### SCHEDULERS
from buildbot.scheduler import Scheduler
webrtc_scheduler = Scheduler(name='all', branch='trunk',
builderNames=['Win32Debug',
'Win32Release',
'MacOS32DBG',
'MacOS32Release',
'Linux32DBG',
'Linux32Release',
'Linux64DBG',
'Linux64Release',
'LinuxClang',
'LinuxMemcheck',
'LinuxTsan',
'LinuxAsan',
'Linux64DBG-GCC4.6',
'LinuxLargeTests',
'MacLargeTests',
'WinLargeTests',
'Android',
'AndroidNDK',
'ChromeOS',
])
# Note that additional schedulers (used by the Chrome+WebRTC builders) are
# defined in master_builders_cfg.py. This is an in progress change to make all
# our bots use the Chromium Buildbot scripts.
c['schedulers'] = [webrtc_scheduler]
####### TESTS
# Defines the supported tests followed by a tuple defining if the tests are
# enabled on Linux, Mac and/or Windows (in that order; defined in utils.py).
from webrtc_buildbot import utils
# Normal tests run on Virtual machines for Linux and Windows and physical
# machines for Mac.
NORMAL_TESTS = {
# Test name Linux Mac Windows
'audio_coding_module_test': (True, True, True),
'audio_coding_unittests': (True, True, True),
'audio_conference_mixer_unittests':(True, True, True),
'audio_device_test_api': (False, True, False), # no audio devices
'audioproc_unittest': (True, True, True),
'cng_unittests': (True, True, True),
'g711_unittests': (True, True, True),
'g722_unittests': (True, True, True),
'libyuv_unittests': (True, True, True),
'jpeg_unittests': (True, True, True),
'media_file_unittests': (True, True, True),
'metrics_unittests': (True, True, True),
'neteq_unittests': (True, True, True),
'pcm16b_unittests': (True, True, True),
'resampler_unittests': (True, True, True),
'rtp_rtcp_unittests': (True, True, True),
'signal_processing_unittests': (True, True, True),
'system_wrappers_unittests': (True, True, True),
'test_bwe': (True, True, True),
'test_fec': (True, True, True),
'test_support_unittests': (True, True, True),
'udp_transport_unittests': (True, True, True),
'vad_unittests': (True, True, True),
'video_codecs_test_framework_integrationtests': (True, True, True),
'video_codecs_test_framework_unittests': (True, True, True),
'video_coding_unittests': (True, True, True),
'video_engine_core_unittests': (True, True, True),
'video_processing_unittests': (True, True, True),
'voice_engine_unittests': (True, True, True),
'vp8_unittests': (True, True, True),
'webrtc_utility_unittests': (True, True, True),
}
# Physical machine tests run on hardware with audio device and webcam present.
PHYSICAL_MACHINE_TESTS = {
# Test name Linux Mac Windows
'audio_device_test_api': (False, False, False), # Issue 257+339
'video_render_module_test': (True, True, True),
'vie_auto_test': (True, True, True),
'voe_auto_test': (True, True, True),
}
MEMCHECK_DISABLED_TESTS = [
'audio_coding_module_test', # Issue 270
'test_fec', # Too slow for Valgrind memcheck
]
TSAN_DISABLED_TESTS = [
'audio_coding_module_test', # Issue 283
'audioproc_unittest', # Issue 299
'system_wrappers_unittests', # Issue 300
'udp_transport_unittests', # Issue 536
'video_processing_unittests', # Issue 303
'test_fec', # Too slow for TSAN
]
ASAN_DISABLED_TESTS = [
'audio_coding_module_test', # Issue 281
'neteq_unittests', # Issue 282
]
linux_normal_tests = utils.GetEnabledTests(NORMAL_TESTS, 'Linux')
mac_normal_tests = utils.GetEnabledTests(NORMAL_TESTS, 'Mac')
windows_normal_tests = utils.GetEnabledTests(NORMAL_TESTS, 'Windows')
linux_physical_machine_tests = utils.GetEnabledTests(PHYSICAL_MACHINE_TESTS,
'Linux')
mac_physical_machine_tests = utils.GetEnabledTests(PHYSICAL_MACHINE_TESTS,
'Mac')
windows_physical_machine_tests = utils.GetEnabledTests(PHYSICAL_MACHINE_TESTS,
'Windows')
####### FACTORIES
CHROME_SVN_URL = 'http://src.chromium.org/svn/trunk/src'
CHROME_LKGR_URL = 'http://chromium-status.appspot.com/lkgr'
CHROME_GCLIENT_SOLUTION_NAME='src'
CHROME_CUSTOM_DEPS_LIST = [
('src/third_party/webrtc', 'http://webrtc.googlecode.com/svn/stable/src'),
('src/third_party/WebKit/LayoutTests', None),
('src/webkit/data/layout_tests/LayoutTests', None),
]
VALGRIND_DEPS_LIST = [
('trunk/third_party/valgrind',
'http://src.chromium.org/svn/trunk/deps/third_party/valgrind/binaries'),
]
ASAN_DEPS_LIST = [
('trunk/third_party/asan',
'http://src.chromium.org/chrome/trunk/deps/third_party/asan'),
]
# Linux
linux_factory_64_dbg = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_factory_64_dbg'))
linux_factory_64_dbg.EnableBuild()
linux_factory_64_dbg.EnableTests(linux_normal_tests)
linux_factory_32_release = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_factory_32_release'))
linux_factory_32_release.EnableBuild(release=True, build32=True)
linux_factory_32_release.EnableTests(linux_normal_tests)
linux_factory_64_release = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_factory_64_release'))
linux_factory_64_release.EnableBuild(release=True)
linux_factory_64_release.EnableTests(linux_normal_tests)
linux_factory_32_dbg = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_factory_32_dbg'))
linux_factory_32_dbg.EnableCoverage(
coverage_url='http://webrtc-cb-linux-slave-4.cbf.corp.google.com/coverage/')
linux_factory_32_dbg.EnableBuild(build32=True)
linux_factory_32_dbg.EnableTests(linux_normal_tests)
# The video factory needs to prepare for valgrind since some tests will run
# under tooling. Not all tests will though since that would be too slow.
linux_factory_large_tests = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_factory_large_tests'),
custom_deps_list=VALGRIND_DEPS_LIST)
linux_factory_large_tests.EnableCoverage(
coverage_url='http://webrtc-build-bot-se.lul/coverage/')
linux_factory_large_tests.EnableBuild(compile_for_memory_tooling=True)
linux_factory_large_tests.EnableTests(linux_physical_machine_tests)
chromeos_factory = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('chromeos_factory'))
chromeos_factory.EnableBuild(chrome_os=True)
chromeos_factory.EnableTests(linux_normal_tests)
linux_clang = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_clang'))
linux_clang.EnableBuild(clang=True)
linux_clang.EnableTests(linux_normal_tests)
linux_memcheck_factory = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_memcheck_factory'), run_with_memcheck=True,
custom_deps_list=VALGRIND_DEPS_LIST)
linux_memcheck_factory.EnableBuild(release=True,
compile_for_memory_tooling=True)
# Filter out disabled memcheck tests:
memcheck_tests = filter(lambda test: test not in MEMCHECK_DISABLED_TESTS,
linux_normal_tests)
linux_memcheck_factory.EnableTests(memcheck_tests)
linux_tsan_factory = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_tsan_factory'), run_with_tsan=True,
custom_deps_list=VALGRIND_DEPS_LIST)
linux_tsan_factory.EnableBuild(release=True, compile_for_memory_tooling=True)
# Filter out disabled tsan tests:
tsan_tests = filter(lambda test: test not in TSAN_DISABLED_TESTS,
linux_normal_tests)
linux_tsan_factory.EnableTests(tsan_tests)
linux_asan_factory = utils.WebRTCLinuxFactory(
utils.BuildStatusOracle('linux_asan_factory'), run_with_asan=True,
custom_deps_list=ASAN_DEPS_LIST)
linux_asan_factory.EnableBuild(release=True, clang=True)
# Filter out disabled asan tests:
asan_tests = filter(lambda test: test not in ASAN_DISABLED_TESTS,
linux_normal_tests)
linux_asan_factory.EnableTests(asan_tests)
android_factory = utils.WebRTCAndroidFactory(
utils.BuildStatusOracle('android_factory'))
android_factory.EnableBuild(product='toro')
android_ndk_factory = utils.WebRTCAndroidNDKFactory(
utils.BuildStatusOracle('android_ndk_factory'))
android_ndk_factory.EnableBuild()
# Mac
mac_factory_32_dbg = utils.WebRTCMacFactory(
utils.BuildStatusOracle('mac_factory_32_dbg'))
mac_factory_32_dbg.EnableBuild(build_type='both')
mac_factory_32_dbg.EnableTests(mac_normal_tests)
mac_factory_32_release = utils.WebRTCMacFactory(
utils.BuildStatusOracle('mac_factory_32_release'))
mac_factory_32_release.EnableBuild(build_type='both', release=True)
mac_factory_32_release.EnableTests(mac_normal_tests)
mac_factory_large_tests = utils.WebRTCMacFactory(
utils.BuildStatusOracle('mac_factory_large_tests'))
mac_factory_large_tests.EnableBuild(build_type='make')
mac_factory_large_tests.EnableTests(mac_physical_machine_tests)
# Windows
win_factory_32_Debug = utils.WebRTCWinFactory(
utils.BuildStatusOracle('win_factory_32_debug'))
win_factory_32_Debug.EnableBuild(configuration='Debug')
win_factory_32_Debug.EnableTests(windows_normal_tests)
win_factory_32_Release = utils.WebRTCWinFactory(
utils.BuildStatusOracle('win_factory_32_release'))
win_factory_32_Release.EnableBuild(configuration='Release')
win_factory_32_Release.EnableTests(windows_normal_tests)
win_factory_large_tests = utils.WebRTCWinFactory(
utils.BuildStatusOracle('win_factory_large_tests'))
win_factory_large_tests.EnableBuild()
win_factory_large_tests.EnableTests(windows_physical_machine_tests)
####### BUILDERS
linux_builder_64_debug = {
'name': 'Linux64DBG',
'slavename': 'webrtc-cb-linux-slave-1',
'builddir': 'linux-slave-1',
'factory': linux_factory_64_dbg,
}
linux_builder_32_release = {
'name': 'Linux32Release',
'slavename': 'webrtc-cb-linux-slave-2',
'builddir': 'linux-slave-2',
'factory': linux_factory_32_release,
}
linux_builder_64_release = {
'name': 'Linux64Release',
'slavename': 'webrtc-cb-linux-slave-5',
'builddir': 'linux-slave-5',
'factory': linux_factory_64_release,
}
linux_builder_32_debug = {
'name': 'Linux32DBG',
'slavename': 'webrtc-cb-linux-slave-4',
'builddir': 'linux-slave-4',
'factory': linux_factory_32_dbg,
}
mac_builder_32_debug = {
'name': 'MacOS32DBG',
'slavename': 'dhcp-172-28-249-144',
'builddir': 'mac-slave-3',
'factory': mac_factory_32_dbg,
}
mac_builder_32_release = {
'name': 'MacOS32Release',
'slavename': 'dhcp-172-28-249-167',
'builddir': 'mac-slave-2',
'factory': mac_factory_32_release,
}
mac_builder_large_tests = {
'name': 'MacLargeTests',
'slavename': 'dhcp-172-28-249-168',
'builddir': 'mac-large-tests',
'factory': mac_factory_large_tests,
}
chromeos_builder = {
'name': 'ChromeOS',
'slavename': 'webrtc-cb-linux-slave-3',
'builddir': 'chromeos',
'factory': chromeos_factory,
}
win_builder_32_debug = {
'name': 'Win32Debug',
'slavename': 'webrtc-win2k8-1',
'builddir': 'win-32-dbg',
'factory': win_factory_32_Debug,
}
win_builder_32_release = {
'name': 'Win32Release',
'slavename': 'webrtc-win2k8-2',
'builddir': 'win-32-release',
'factory': win_factory_32_Release,
}
win_builder_large_tests = {
'name': 'WinLargeTests',
'slavename': 'webrtc-test-w',
'builddir': 'win-large-tests',
'factory': win_factory_large_tests,
}
linux_builder_large_tests = {
'name': 'LinuxLargeTests',
'slavename': 'webrtc-build-bot-se',
'builddir': 'linux-large-tests',
'factory': linux_factory_large_tests,
}
linux_builder_clang = {
'name': 'LinuxClang',
'slavename': 'webrtc-cb-linux-slave-8',
'builddir': 'linux-clang',
'factory': linux_clang,
}
linux_builder_memcheck = {
'name': 'LinuxMemcheck',
'slavename': 'webrtc-cb-linux-slave-6',
'builddir': 'linux-memcheck',
'factory': linux_memcheck_factory,
}
linux_builder_tsan = {
'name': 'LinuxTsan',
'slavename': 'webrtc-cb-linux-slave-15',
'builddir': 'linux-tsan',
'factory': linux_tsan_factory,
}
linux_builder_asan = {
'name': 'LinuxAsan',
'slavename': 'webrtc-cb-linux-slave-16',
'builddir': 'linux-asan',
'factory': linux_asan_factory,
}
android_builder_1 = {
'name': 'Android',
'slavename': 'webrtc-cb-linux-slave-7',
'builddir': 'android',
'factory': android_factory,
}
android_builder_ndk = {
'name': 'AndroidNDK',
'slavename': 'webrtc-cb-linux-slave-10',
'builddir': 'android-ndk',
'factory': android_ndk_factory,
}
linux_builder_gcc_4_6 = {
'name': 'Linux64DBG-GCC4.6',
'slavename': 'webrtc-cb-linux-slave-9',
'builddir': 'linux-slave-gcc-4.6',
'factory': linux_factory_64_dbg,
}
c['builders'] = [
win_builder_32_debug,
win_builder_32_release,
mac_builder_32_debug,
mac_builder_32_release,
linux_builder_32_debug,
linux_builder_32_release,
linux_builder_64_release,
linux_builder_64_debug,
linux_builder_clang,
linux_builder_memcheck,
linux_builder_tsan,
linux_builder_asan,
linux_builder_gcc_4_6,
win_builder_large_tests,
mac_builder_large_tests,
linux_builder_large_tests,
android_builder_1,
android_builder_ndk,
chromeos_builder,
]
# Load Chrome bots that are using Chromium factories.
# These modules come from the local directory.
import master_builders_cfg
master_builders_cfg.ConfigureChromeWebRTCBuilders(c)
master_builders_cfg.ConfigureNightlyChromeWebRTCBloatBuilder(c)
####### BUILDSLAVES
# Slaves are loaded from slaves.cfg.
slaves = slaves_list.SlavesList('slaves.cfg', 'WebRTC')
for builder in c['builders']:
# Associate the slaves to the builders. The configuration is in slaves.cfg.
builder['slavenames'] = slaves.GetSlavesName(builder=builder['name'])
# The 'slaves' list defines the set of allowable buildslaves. List all the
# slaves registered to a builder. Remove dupes.
c['slaves'] = master_utils.AutoSetupSlaves(c['builders'],
config.Master.GetBotPassword())
# Make sure everything works together.
master_utils.VerifySetup(c, slaves)
# Adds common status and tools to this master.
master_utils.AutoSetupMaster(c, ActiveMaster, mail_notifier=False,
enable_http_status_push=False)
####### STATUS TARGETS
# Override the tgrid URL since we're parsing that in WebRTC for LKGR
# (Chrome doesn't use it and puts the Console page on that URL instead).
c['status'][0].putChild("tgrid", grid.TransposedGridStatusResource())
# Use an environment variable to easily avoid enabling e-mail for development.
if not os.getenv('BUILDBOT_DEVELOPMENT_MODE'):
email_status = mail.MailNotifier(
fromaddr='webrtc-cb-watchlist@google.com',
extraRecipients=['webrtc-cb-watchlist@google.com'],
sendToInterestedUsers=True,
mode='failing',
lookup=master_utils.UsersAreEmails())
c['status'] += [email_status]
####### DB URL
# This specifies what database buildbot uses to store change and scheduler
# state. You can leave this at its default for all but the largest
# installations.
c['db_url'] = 'sqlite:///state.sqlite'
####### DEBUGGING OPTIONS
# if you set 'debugPassword', then you can connect to the buildmaster with
# the diagnostic tool in contrib/debugclient.py . From this tool, you can
# manually force builds and inject changes, which may be useful for testing
# your buildmaster without actually committing changes to your repository (or
# before you have a functioning 'sources' set up). The debug tool uses the
# same port number as the slaves do: 'slavePortnum'.
#c['debugPassword'] = 'debugpassword'
# if you set 'manhole', you can ssh into the buildmaster and get an
# interactive python shell, which may be useful for debugging buildbot
# internals. It is probably only useful for buildbot developers. You can also
# use an authorized_keys file, or plain telnet.
#from buildbot import manhole
#c['manhole'] = manhole.PasswordManhole('tcp:9999:interface=127.0.0.1',
# 'admin', 'password')
Reference in New Issue View Git Blame Copy Permalink