webrtc/tools/continuous_build/webrtc_buildbot/utils.py

626 lines
25 KiB
Python
Raw Normal View History

#!/usr/bin/env python
# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
__author__ = 'ivinnichenko@webrtc.org (Illya Vinnichenko)'
from buildbot.process import factory
from buildbot.steps import shell
from buildbot.steps.shell import ShellCommand
from buildbot.process import properties
from buildbot.process.properties import WithProperties
import os
import sys
SVN_LOCATION = "http://webrtc.googlecode.com/svn/trunk"
class WebRTCFactory(factory.BuildFactory):
"""A Build Factory affected by properties."""
def __init__(self, build_factory_properties=None, steps=None,
enable_coverage=False, account=None):
factory.BuildFactory.__init__(self, steps)
self.properties = properties.Properties()
self.enable_build = False
self.force_sync = False
self.headless_tests = []
self.enable_coverage = enable_coverage
self.gyp_params = []
self.account = account
self.coverage_dir = ""
self.coverage_url = ""
self.release = False
if self.account is None:
self.coverage_url = "http://www.corp.google.com/~webrtc-cb"
self.coverage_dir = "/home/webrtc-cb/www"
else:
self.coverage_url = "http://www.corp.google.com/~%s" % self.account
self.coverage_dir = "/home/%s/www" % self.account
def EnableBuild(self, force_sync):
"""Build the binary [must be overridden]."""
pass
def AddCommonTestSteps(self, test):
"""Add common steps for test.
test: test to be run.
"""
self.AddCommonMakeStep(test)
self.AddCommonTestRunStep(test)
def AddCommonStep(self, cmd, descriptor="", workdir="build",
warnOnFailure=False,):
"""Define common step."""
if type(descriptor) is str:
descriptor = [descriptor]
warn = warnOnFailure
flunkOnFailure = not warn
self.addStep(shell.ShellCommand(command=cmd, workdir=workdir,
description=descriptor+["running..."],
descriptionDone=descriptor+["done..."],
warnOnFailure=warn,
flunkOnFailure=flunkOnFailure,
name="".join(descriptor)))
def AddCommonTestRunStep(self):
"""Define common test run step [must be overridden]."""
pass
def AddCommonMakeStep(self):
"""Define common make step [must be overridden]."""
pass
def AddCommonGYPStep(self, gyp_file, gyp_params=[], descriptor="gyp"):
cmd = ["./build/gyp_chromium", "--depth=.", gyp_file]
cmd += gyp_params + self.gyp_params
self.addStep(shell.ShellCommand(command=cmd, workdir="build/trunk",
description=[descriptor, "running..."],
descriptionDone=[descriptor, "done..."],
name="gyp_%s" % descriptor))
def EnableTest(self, test):
"""Enable Test to be run. [must be overridden]
test: test to be run.
"""
pass
def EnableTests(self, tests):
"""Enable Test to be run.
tests: list of test to be run.
"""
print "Headless tests:%s" % self.headless_tests
if self.enable_coverage:
self.EnableBaseCoverage()
for test in tests:
self.EnableTest(test)
if self.enable_coverage:
self.EnableCoverage()
def EnableHeadLess(self, tests):
"""Add headless (build only) tests.
tests: list of headless test.
"""
self.headless_tests += tests
def EnableBaseCoverage(self):
"""Enable base coverage data [must be overridden]."""
pass
def EnableCoverage(self):
"""Enable coverage data [must be overridden]."""
pass
class GenerateCodeCoverage(ShellCommand):
command = ["genhtml", "final.info", "--output-directory",
WithProperties("/home/webrtc-cb/www/%(buildername)s_%(buildnumber)s")]
name = "LCOV_GenHTML"
warnOnFailure = True
def __init__(self, coverage_url=None, coverage_dir=None, **kwargs):
if coverage_url is None or coverage_dir is None:
raise TypeError("coverage location required")
print coverage_url, coverage_dir
ShellCommand.__init__(self, **kwargs)
self.addFactoryArguments(coverage_url=coverage_url,
coverage_dir=coverage_dir)
self.setDefaultWorkdir("build/trunk")
self.coverage_url = coverage_url
self.coverage_dir = coverage_dir
self.setCommand(["genhtml", "final.info", "--output-directory",
WithProperties(coverage_dir + "/%(buildername)s_%(buildnumber)s")])
def createSummary(self, log):
coverage_url = "%s/%s_%s" % (self.coverage_url,
self.getProperty("buildername"), self.getProperty("buildnumber"))
coverage_dir = "%s/%s_%s" % (self.coverage_dir,
self.getProperty("buildername"), self.getProperty("buildnumber"))
os.chmod(coverage_dir,0777)
for root, dirs, files in os.walk(coverage_dir):
for d in dirs:
os.chmod(os.path.join(root, d), 0777)
for f in files:
os.chmod(os.path.join(root, f), 0777)
self.addURL("coverage", coverage_url)
def start(self):
ShellCommand.start(self)
################################################################################
class WebRTCAndroidFactory(WebRTCFactory):
"""A Build Factory affected by properties."""
def __init__(self, build_factory_properties=None, steps=None,
enable_coverage=False, account=None):
WebRTCFactory.__init__(self, build_factory_properties, steps,
enable_coverage, account)
def EnableBuild(self, product="toro"):
prefix = "rm -rf out/target/product/%s/obj/" % product
cleanup_list = [
"rm -rf external/webrtc",
prefix + "STATIC_LIBRARIES/libwebrtc_*",
prefix + "SHARE_LIBRARIES/libwebrtc_*",
prefix + "EXECUTABLES/webrtc_*"
]
cmd = " ; ".join(cleanup_list)
self.addStep(shell.Compile(command=(cmd), workdir="build/trunk",
description=["cleanup", "running..."], haltOnFailure=False,
warnOnFailure=True, flunkOnFailure =False,
descriptionDone=["cleanup", "done..."], name="cleanup"))
cmd = "svn checkout http://webrtc.googlecode.com/svn/trunk/ external/webrtc"
self.addStep(shell.Compile(command=(cmd),
workdir="build/trunk", description=["svn", "running..."],
haltOnFailure=False, descriptionDone=["svn", "done..."], name="svn"))
cmd = ("source build/envsetup.sh && lunch full_%s-eng "
"&& mmm external/webrtc showcommands" % product)
self.addStep(shell.Compile(command=(cmd),
workdir="build/trunk", description=["build", "running..."],
haltOnFailure=False,
descriptionDone=["build", "done..."], name="build"))
################################################################################
################################################################################
class WebRTCChromeFactory(WebRTCFactory):
"""A Build Factory affected by properties."""
def __init__(self, build_factory_properties=None, steps=None,
enable_coverage=False, account=None):
WebRTCFactory.__init__(self, build_factory_properties, steps,
enable_coverage, account)
def EnableBuild(self):
self.AddCommonStep(["rm", "-rf", "src"], descriptor="Cleanup")
cmd = ["gclient", "sync", "--force"]
self.AddCommonStep(cmd, descriptor="Sync")
self.AddCommonMakeStep("chrome")
def AddCommonMakeStep(self, make, descriptor="", make_extra=None):
make_descriptor = [make, descriptor]
cmd = ["make", make, "-j100"]
if make_extra is not None:
cmd.append(make_extra)
self.addStep(shell.ShellCommand(command=cmd,
workdir="build/src", description=["Making"]+make_descriptor,
descriptionDone=make_descriptor+["built"],
name="_".join(make_descriptor)))
################################################################################
class WebRTCLinuxFactory(WebRTCFactory):
"""A Build Factory affected by properties."""
def __init__(self, build_factory_properties=None, steps=None,
enable_coverage=False, account=None):
WebRTCFactory.__init__(self, build_factory_properties, steps,
enable_coverage, account)
def EnableBuild(self, force_sync=False, release=False, build32=False,
chrome_os=False, clang=False):
if build32:
self.gyp_params.append("-Dtarget_arch=ia32")
self.enable_build = True
self.force_sync = force_sync
"""Linux specific Build"""
self.release = release
self.AddCommonStep(["rm", "-rf", "trunk"], descriptor="Cleanup")
self.AddCommonStep(["gclient", "config", SVN_LOCATION],
descriptor="gclient_config")
cmd = ["gclient", "sync"]
if force_sync:
cmd.append("--force")
self.AddCommonStep(cmd, descriptor="Sync")
if chrome_os:
self.gyp_params.append("-Dchromeos=1")
if clang:
self.gyp_params.append("-Dclang=1")
if self.enable_coverage:
self.gyp_params.append("-Dcoverage=1")
self.AddCommonGYPStep("webrtc.gyp", descriptor="CommonGYP")
if clang:
self.AddCommonStep(["trunk/tools/clang/scripts/update.sh"],
descriptor="Update_Clang")
if self.release:
self.AddCommonMakeStep("all", make_extra="BUILDTYPE=Release")
else:
self.AddCommonMakeStep("all")
def AddCommonTestRunStep(self, test, descriptor="", cmd=None,
workdir="build/trunk"):
if test in self.headless_tests:
return
test_folder = "Release" if self.release else "Debug"
test_descriptor = [test, descriptor]
if cmd is None:
cmd = ["out/%s/%s" % (test_folder, test)]
self.addStep(shell.ShellCommand(command=cmd,
workdir=workdir, description=["Running"]+test_descriptor,
descriptionDone=test_descriptor+["finished"],
name="_".join(test_descriptor)))
def AddCommonMakeStep(self, make, descriptor="", make_extra=None):
make_descriptor = [make, descriptor]
#cpu = `grep -i \"processor\" /proc/cpuinfo | sort -u | wc -l`
cmd = ["make", make, "-j100"]
if make_extra is not None:
cmd.append(make_extra)
self.addStep(shell.ShellCommand(command=cmd,
workdir="build/trunk", description=["Making"]+make_descriptor,
descriptionDone=make_descriptor+["built"],
name="_".join(make_descriptor)))
def EnableBaseCoverage(self):
self.AddCommonStep(["lcov", "--directory", ".", "--zerocounters"],
workdir="build/trunk",
warnOnFailure=True,
descriptor=["LCOV", "Zero"])
self.AddCommonStep(["lcov", "--directory", ".", "--capture", "-b",
".", "--initial",
"--output-file", "webrtc_base.info"],
workdir="build/trunk",
warnOnFailure=True,
descriptor=["LCOV", "Base", "Capture"])
self.AddCommonStep(['lcov', '--extract', 'webrtc_base.info', '*/src/*',
'--output', 'filtered.info'],
workdir="build/trunk",
warnOnFailure=True,
descriptor=["LCOV", "Base", "Extract"])
self.AddCommonStep(["lcov", "--remove", "filtered.info", "*/usr/include/*",
"/third*", "/testing/*", "*/test/*", "*_unittest.*",
"*/mock/*", "--output",
"webrtc_base_filtered_final.info"],
workdir="build/trunk",
warnOnFailure=True,
descriptor=["LCOV", "Base", "Filter"])
def EnableCoverage(self):
"""Enable coverage data."""
self.AddCommonStep(["lcov", "--directory", ".", "--capture", "-b",
".", "--output-file", "webrtc.info"],
warnOnFailure=True,
workdir="build/trunk", descriptor=["LCOV", "Capture"])
self.AddCommonStep(['lcov', '--extract', 'webrtc.info', '*/src/*',
'--output', 'test.info'], warnOnFailure=True,
workdir="build/trunk", descriptor=["LCOV", "Extract"])
self.AddCommonStep(["lcov", "--remove", "test.info", "*/usr/include/*",
"/third*", "/testing/*", "*/test/*", "*_unittest.*",
"*/mock/*", "--output",
"final.info"], warnOnFailure=True,
workdir="build/trunk", descriptor=["LCOV", "Filter"])
self.AddCommonStep(['lcov', '-a', 'webrtc_base_filtered_final.info', '-a',
'final.info', '-o', 'final.info'], warnOnFailure=True,
workdir="build/trunk", descriptor=["LCOV", "Merge"])
self.addStep(GenerateCodeCoverage(coverage_url=self.coverage_url,
coverage_dir=self.coverage_dir))
def EnableTest(self, test):
"""Enable Test to be run.
test: test to be run.
"""
if test == "audioproc_unittest":
self.AddCommonTestRunStep(test)
self.AddCommonGYPStep("webrtc.gyp", gyp_params=["-Dprefer_fixed_point=1"],
descriptor="fixed_point")
self.AddCommonMakeStep(test, descriptor="make_fixed_point")
self.AddCommonTestRunStep(test, descriptor="fixed_point")
elif test == "signal_processing_unittests":
self.AddCommonTestRunStep(test)
elif test == "resampler_unittests":
self.AddCommonTestRunStep(test)
elif test == "vad_unittests":
self.AddCommonTestRunStep(test)
elif test == "rtp_rtcp_unittests":
self.AddCommonTestRunStep(test)
elif test == "video_coding_unittests":
self.AddCommonTestRunStep(test)
elif test == "test_bwe":
self.AddCommonTestRunStep(test)
elif test == "audio_device_test_api":
self.AddCommonTestRunStep(test)
elif test == "audio_device_test_func":
self.AddCommonTestRunStep(test)
elif test == "audio_coding_module_test":
self.AddCommonTestRunStep(test)
elif test == "video_processing_unittests":
self.AddCommonTestRunStep(test)
elif test == "test_fec":
self.AddCommonTestRunStep(test)
elif test == "system_wrappers_unittests":
self.AddCommonTestRunStep(test)
elif test == "cng_unittests":
self.AddCommonTestRunStep(test)
elif test == "g711_unittests":
self.AddCommonTestRunStep(test)
elif test == "g722_unittests":
self.AddCommonTestRunStep(test)
elif test == "pcm16b_unittests":
self.AddCommonTestRunStep(test)
elif test == "audio_conference_mixer_unittests":
self.AddCommonTestRunStep(test)
elif test == "media_file_unittests":
self.AddCommonTestRunStep(test)
elif test == "udp_transport_unittests":
self.AddCommonTestRunStep(test)
elif test == "webrtc_utility_unittests":
self.AddCommonTestRunStep(test)
elif test == "neteq_unittests":
self.AddCommonTestRunStep(test)
elif test == "vp8_unittests":
self.AddCommonTestRunStep(test)
elif test == "libyuv_unittests":
self.AddCommonTestRunStep(test)
elif test == "voice_engine_unittests":
self.AddCommonTestRunStep(test)
elif test == "video_engine_core_unittests":
self.AddCommonTestRunStep(test)
elif test == "audio_coding_unittests":
self.AddCommonTestRunStep(test)
elif test == "vie_auto_test":
self.addStep(shell.Compile(command=('xvfb-run --server-args="-screen 0 '
'800x600x24 -extension Composite" out/Debug/vie_auto_test --automated '
'--gtest_filter="ViEStandardIntegrationTest.*:ViEApiIntegrationTest.*" '
'--capture_test_ensure_resolution_alignment_in_capture_device=false'),
workdir="build/trunk", description=[test, "running..."],
descriptionDone=[test, "done..."], name="%s" % test))
elif test == "voe_auto_test":
self.addStep(shell.Compile(command=('out/Debug/voe_auto_test --automated '
'--gtest_filter="-*Manual*"'),
workdir="build/trunk", description=[test, "running..."],
descriptionDone=[test, "done..."], name="%s" % test))
else:
print "[Linux]: No supported tests are found for [%s]" % test
################################################################################
class WebRTCMacFactory(WebRTCFactory):
"""A Build Factory affected by properties."""
def __init__(self, build_factory_properties=None, steps=None,
enable_coverage=False, account=None):
WebRTCFactory.__init__(self, build_factory_properties, steps,
enable_coverage, account)
self.build_type = "both"
self.allowed_build_types = ["both", "xcode", "make"]
def EnableBuild(self, force_sync=True, build_type="both", release=False):
self.enable_build = True
self.force_sync = force_sync
self.release = release
"""Mac specific Build"""
if build_type not in self.allowed_build_types:
print "*** INCORRECT BUILD TYPE (%s)!!! ***" % build_type
sys.exit(0)
else:
self.build_type = build_type
self.AddCommonStep(["rm", "-rf", "trunk"], descriptor="Cleanup")
self.AddCommonStep(["gclient", "config", SVN_LOCATION],
descriptor="gclient_config")
cmd = ["gclient", "sync"]
if force_sync:
cmd.append("--force")
self.AddCommonStep(cmd, descriptor="Sync")
if self.build_type == "make" or self.build_type == "both":
self.AddCommonGYPStep("webrtc.gyp", gyp_params=["-f", "make"],
descriptor="EnableMake")
self.AddCommonMakeStep("all")
def AddCommonTestRunStep(self, test, descriptor="", cmd=None,
workdir="build/trunk"):
if test in self.headless_tests:
return
test_folder = "Release" if self.release else "Debug"
test_descriptor = [test, descriptor]
if cmd is None:
if self.build_type == "xcode" or self.build_type == "both":
cmd = ["xcodebuild/%s/%s" % (test_folder, test)]
self.AddCommonStep(cmd, descriptor=test_descriptor+["(xcode)"],
workdir="build/trunk")
if self.build_type == "make" or self.build_type == "both":
cmd = ["out/%s/%s" % (test_folder, test)]
self.AddCommonStep(cmd, descriptor=test_descriptor+["(make)"],
workdir="build/trunk")
def AddCommonMakeStep(self, make, descriptor="", make_extra=None):
make_descriptor = [make, descriptor]
cpu = "`sysctl -n hw.logicalcpu`"
if self.build_type == "make" or self.build_type == "both":
cmd = ["make", make, "-j100"]
if make_extra is not None:
cmd.append(make_extra)
self.AddCommonStep(cmd, descriptor=make_descriptor+["(make)"],
workdir="build/trunk")
if self.build_type == "xcode" or self.build_type == "both":
cmd = ["xcodebuild", "-project", "webrtc.xcodeproj", "-configuration",
"Debug", "-target", "All"]
self.AddCommonStep(cmd, descriptor=make_descriptor+["(xcode)"],
workdir="build/trunk")
def EnableTest(self, test):
"""Enable Test to be run.
test: test to be run.
"""
if test == "audioproc_unittest":
self.AddCommonTestRunStep(test)
elif test == "signal_processing_unittests":
self.AddCommonTestRunStep(test)
elif test == "resampler_unittests":
self.AddCommonTestRunStep(test)
elif test == "vad_unittests":
self.AddCommonTestRunStep(test)
elif test == "rtp_rtcp_unittests":
self.AddCommonTestRunStep(test)
elif test == "video_coding_unittests":
self.AddCommonTestRunStep(test)
elif test == "test_bwe":
self.AddCommonTestRunStep(test)
elif test == "audio_device_test_api":
self.AddCommonTestRunStep(test)
elif test == "audio_device_test_func":
self.AddCommonTestRunStep(test)
elif test == "audio_coding_module_test":
self.AddCommonTestRunStep(test)
elif test == "video_processing_unittests":
self.AddCommonTestRunStep(test)
elif test == "test_fec":
self.AddCommonTestRunStep(test)
elif test == "system_wrappers_unittests":
self.AddCommonTestRunStep(test)
elif test == "audio_conference_mixer_unittests":
self.AddCommonTestRunStep(test)
elif test == "media_file_unittests":
self.AddCommonTestRunStep(test)
elif test == "udp_transport_unittests":
self.AddCommonTestRunStep(test)
elif test == "webrtc_utility_unittests":
self.AddCommonTestRunStep(test)
elif test == "neteq_unittests":
self.AddCommonTestRunStep(test)
elif test == "vp8_unittests":
self.AddCommonTestRunStep(test)
elif test == "libyuv_unittests":
self.AddCommonTestRunStep(test)
elif test == "voice_engine_unittests":
self.AddCommonTestRunStep(test)
elif test == "video_engine_core_unittests":
self.AddCommonTestRunStep(test)
elif test == "audio_coding_unittests":
self.AddCommonTestRunStep(test)
else:
print "[Mac]: No supported tests are found for [%s]" % test
################################################################################
class WebRTCWinFactory(WebRTCFactory):
"""A Build Factory affected by properties."""
def __init__(self, build_factory_properties=None, steps=None,
enable_coverage=False, account=None):
WebRTCFactory.__init__(self, build_factory_properties, steps,
enable_coverage, account)
self.configuration = "Debug"
self.platform = "x64"
self.allowed_platforms = ["x64", "Win32"]
self.allowed_configurations = ["Debug", "Release", "both"]
def EnableBuild(self, force_sync=True, platform="Win32",
configuration="Debug", build_only=False):
self.enable_build = True
self.force_sync = force_sync
"""Win specific Build"""
if platform not in self.allowed_platforms:
print "*** INCORRECT PLATFORM (%s)!!! ***" % platform
sys.exit(0)
else:
self.platform = platform
if configuration not in self.allowed_configurations:
print "*** INCORRECT CONFIGURATION (%s)!!! ***" % configuration
sys.exit(0)
else:
self.configuration = configuration
if not build_only:
self.AddCommonStep(["rm", "-rf", "trunk"], descriptor="Cleanup")
self.AddCommonStep(["gclient", "config", SVN_LOCATION],
descriptor="gclient_config")
cmd = ["gclient", "sync"]
if force_sync:
cmd.append("--force")
self.AddCommonStep(cmd, descriptor="Sync")
if self.configuration == "Debug" or self.configuration == "both":
cmd = ["msbuild", "webrtc.sln", "/t:Clean",
"/p:Configuration=Debug;Platform=%s" % (self.platform)]
self.AddCommonStep(cmd, descriptor="Build_Clean", workdir="build/trunk")
cmd = ["msbuild", "webrtc.sln",
"/p:Configuration=Debug;Platform=%s" % (self.platform)]
self.AddCommonStep(cmd, descriptor="Build_Debug", workdir="build/trunk")
if self.configuration == "Release" or self.configuration == "both":
cmd = ["msbuild", "webrtc.sln", "/t:Clean",
"/p:Configuration=Release;Platform=%s" % (self.platform)]
self.AddCommonStep(cmd, descriptor="Build_Clean", workdir="build/trunk")
cmd = ["msbuild", "webrtc.sln",
"/p:Configuration=Release;Platform=%s" % (self.platform)]
self.AddCommonStep(cmd, descriptor="Build_Release", workdir="build/trunk")
def AddCommonTestRunStep(self, test, descriptor="", cmd=None,
workdir="build/trunk"):
if test in self.headless_tests:
return
test_descriptor = [test, descriptor]
if cmd is None:
if self.configuration == "Debug" or self.configuration == "both":
cmd = ["build\Debug\%s.exe" % test]
self.AddCommonStep(cmd, descriptor=test_descriptor+["Debug"],
workdir=workdir)
if self.configuration == "Release" or self.configuration == "both":
cmd = ["build\Release\%s.exe" % test]
self.AddCommonStep(cmd, descriptor=test_descriptor+["Release"],
workdir=workdir)
def EnableTest(self, test):
"""Enable Test to be run.
test: test to be run.
"""
if test == "audioproc_unittest":
self.AddCommonTestRunStep(test)
elif test == "resampler_unittests":
self.AddCommonTestRunStep(test)
elif test == "vad_unittests":
self.AddCommonTestRunStep(test)
elif test == "system_wrappers_unittests":
self.AddCommonTestRunStep(test)
elif test == "neteq_unittests":
self.AddCommonTestRunStep(test)
elif test == "vp8_unittests":
self.AddCommonTestRunStep(test)
elif test == "libyuv_unittests":
self.AddCommonTestRunStep(test)
elif test == "voice_engine_unittests":
self.AddCommonTestRunStep(test)
elif test == "video_engine_core_unittests":
self.AddCommonTestRunStep(test)
elif test == "audio_coding_unittests":
self.AddCommonTestRunStep(test)
else:
print "[Win]: No supported tests are found for [%s]" % test