Implemented bloat calculation. This will measure the binary size of Chrome+WebRTC components each weekend.
BUG= TEST= Review URL: https://webrtc-codereview.appspot.com/508005 git-svn-id: http://webrtc.googlecode.com/svn/trunk@2088 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
parent
39946f1380
commit
f6cd33dd89
@ -26,6 +26,12 @@ deps = {
|
||||
# Used by tools/quality_tracking/dashboard and tools/python_charts.
|
||||
"tools/third_party/google-visualization-python":
|
||||
"http://google-visualization-python.googlecode.com/svn/trunk/@15",
|
||||
|
||||
# Used by tools/continuous_build/build_internal/symsrc/calculate_bloat.py.
|
||||
"tools/third_party/bloat":
|
||||
"https://github.com/martine/bloat.git@31428aaa491",
|
||||
"tools/third_party/webtreemap":
|
||||
"https://github.com/martine/webtreemap.git@7839cf9154",
|
||||
}
|
||||
|
||||
hooks = [
|
||||
|
@ -17,6 +17,7 @@ from buildbot.changes.pb import PBChangeSource
|
||||
from buildbot.changes.svnpoller import SVNPoller
|
||||
from buildbot.process import factory
|
||||
from buildbot.scheduler import Scheduler
|
||||
from buildbot.schedulers import timed
|
||||
from buildbot.status import html
|
||||
from buildbot.status import mail
|
||||
from buildbot.steps import shell
|
||||
@ -54,10 +55,20 @@ webrtc_scheduler = Scheduler(name='all', branch=None, treeStableTimer=5*60,
|
||||
'AndroidNDK',
|
||||
'ChromeOS'
|
||||
])
|
||||
|
||||
chrome_scheduler = Scheduler(name='chrome', branch=None, treeStableTimer=60*60,
|
||||
builderNames=['Chrome'])
|
||||
|
||||
c['schedulers'] = [webrtc_scheduler, chrome_scheduler]
|
||||
# Run the weekend scheduler at sunday, 2 AM CST/CDT. This will mean roughly
|
||||
# Sunday 9 AM in the CET timezone, which should avoid everyone's working hours.
|
||||
weekend_scheduler = timed.Nightly(name='weekend',
|
||||
builderNames=['ChromeBloat'],
|
||||
branch=None,
|
||||
dayOfWeek=6,
|
||||
hour=2,
|
||||
minute=0)
|
||||
|
||||
c['schedulers'] = [webrtc_scheduler, chrome_scheduler, weekend_scheduler]
|
||||
|
||||
####### TESTS
|
||||
# Defines the supported tests followed by a tuple defining if the tests are
|
||||
@ -163,21 +174,33 @@ chromeos_factory = utils.WebRTCLinuxFactory(
|
||||
chromeos_factory.EnableBuild(chrome_os=True)
|
||||
chromeos_factory.EnableTests(linux_normal_tests)
|
||||
|
||||
CHROME_SVN_URL = 'http://src.chromium.org/svn/trunk/src'
|
||||
CHROME_LKGR_URL = 'http://chromium-status.appspot.com/lkgr'
|
||||
CHROME_GCLIENT_SOLUTION_NAME='src'
|
||||
CHROME_CUSTOM_DEPS_LIST = [
|
||||
('src/third_party/webrtc', 'http://webrtc.googlecode.com/svn/stable/src'),
|
||||
('src/third_party/WebKit/LayoutTests', None),
|
||||
('src/chrome/tools/test/reference_build', None),
|
||||
]
|
||||
|
||||
linux_chrome_factory = utils.WebRTCChromeFactory(
|
||||
utils.BuildStatusOracle('linux_chrome'),
|
||||
gclient_solution_name='src',
|
||||
svn_url='http://src.chromium.org/svn/trunk/src',
|
||||
custom_deps_list=[
|
||||
('src/third_party/webrtc',
|
||||
'http://webrtc.googlecode.com/svn/stable/src'),
|
||||
('src/third_party/WebKit/LayoutTests',
|
||||
None),
|
||||
('src/chrome/tools/test/reference_build',
|
||||
None),
|
||||
],
|
||||
safesync_url='http://chromium-status.appspot.com/lkgr')
|
||||
gclient_solution_name=CHROME_GCLIENT_SOLUTION_NAME,
|
||||
svn_url=CHROME_SVN_URL,
|
||||
custom_deps_list=CHROME_CUSTOM_DEPS_LIST,
|
||||
safesync_url=CHROME_LKGR_URL)
|
||||
linux_chrome_factory.EnableBuild()
|
||||
|
||||
linux_chrome_bloat_factory = utils.WebRTCChromeFactory(
|
||||
utils.BuildStatusOracle('linux_chrome_bloat'),
|
||||
gclient_solution_name=CHROME_GCLIENT_SOLUTION_NAME,
|
||||
svn_url=CHROME_SVN_URL,
|
||||
custom_deps_list=CHROME_CUSTOM_DEPS_LIST,
|
||||
safesync_url=CHROME_LKGR_URL)
|
||||
linux_chrome_bloat_factory.EnableBuild(release=True, enable_profiling=True)
|
||||
linux_chrome_bloat_factory.EnableBloatCalculation()
|
||||
|
||||
|
||||
linux_clang = utils.WebRTCLinuxFactory(
|
||||
utils.BuildStatusOracle('linux_clang'))
|
||||
linux_clang.EnableBuild(clang=True)
|
||||
@ -292,6 +315,12 @@ linux_builder_chrome = {
|
||||
'builddir': 'linux-chrome',
|
||||
'factory': linux_chrome_factory,
|
||||
}
|
||||
linux_builder_chrome_bloat = {
|
||||
'name': 'ChromeBloat',
|
||||
'slavename': 'webrtc-chrome',
|
||||
'builddir': 'linux-chrome-bloat',
|
||||
'factory': linux_chrome_bloat_factory,
|
||||
}
|
||||
linux_builder_clang = {
|
||||
'name': 'LinuxClang',
|
||||
'slavename': 'webrtc-cb-linux-slave-8',
|
||||
@ -339,6 +368,7 @@ c['builders'] = [
|
||||
android_builder_ndk,
|
||||
chromeos_builder,
|
||||
linux_builder_chrome,
|
||||
linux_builder_chrome_bloat,
|
||||
]
|
||||
|
||||
|
||||
|
@ -35,6 +35,7 @@ WEBRTC_BUILD_DIR = 'build/'
|
||||
VALGRIND_CMD = ['tools/valgrind-webrtc/webrtc_tests.sh', '-t', 'cmdline']
|
||||
|
||||
DEFAULT_COVERAGE_DIR = '/var/www/coverage/'
|
||||
DEFAULT_BLOAT_DIR = '/var/www/bloat/'
|
||||
DEFAULT_MASTER_WORK_DIR = '.'
|
||||
GCLIENT_RETRIES = 3
|
||||
|
||||
@ -130,7 +131,8 @@ class WebRTCFactory(factory.BuildFactory):
|
||||
self.EnableTest(test)
|
||||
|
||||
def AddCommonStep(self, cmd, descriptor='', workdir=WEBRTC_TRUNK_DIR,
|
||||
halt_build_on_failure=True, warn_on_failure=False):
|
||||
halt_build_on_failure=True, warn_on_failure=False,
|
||||
timeout=1200):
|
||||
"""Adds a step which will run as a shell command on the slave.
|
||||
|
||||
NOTE: you are recommended to use this method to add new shell commands
|
||||
@ -155,6 +157,7 @@ class WebRTCFactory(factory.BuildFactory):
|
||||
warn_on_failure.
|
||||
warn_on_failure: If true, this step isn't that important and will not
|
||||
cause a failed build on failure.
|
||||
timeout: The timeout for the command, in seconds.
|
||||
"""
|
||||
flunk_on_failure = not warn_on_failure
|
||||
|
||||
@ -173,7 +176,8 @@ class WebRTCFactory(factory.BuildFactory):
|
||||
warnOnFailure=warn_on_failure,
|
||||
flunkOnFailure=flunk_on_failure,
|
||||
haltOnFailure=halt_build_on_failure,
|
||||
name='_'.join(descriptor)))
|
||||
name='_'.join(descriptor),
|
||||
timeout=timeout))
|
||||
|
||||
def AddSmartCleanStep(self):
|
||||
"""Adds a smart clean step.
|
||||
@ -230,9 +234,9 @@ class WebRTCFactory(factory.BuildFactory):
|
||||
# Try 4+1=5 times, 10 seconds apart.
|
||||
retry = (10, 4)
|
||||
# Subversion timeout is by default 2 minutes; we allow 5 minutes.
|
||||
timeout = 60*5
|
||||
timeout = 60 * 5
|
||||
# Removal can take a long time. Allow 15 minutes.
|
||||
rm_timeout = 60*15
|
||||
rm_timeout = 60 * 15
|
||||
self.addStep(chromium_step.GClient,
|
||||
gclient_spec=gclient_spec,
|
||||
svnurl=WEBRTC_SVN_LOCATION,
|
||||
@ -495,7 +499,7 @@ class WebRTCAndroidNDKFactory(WebRTCFactory):
|
||||
self.AddCommonStep(cmd=full_cmd, descriptor=descriptor)
|
||||
|
||||
class WebRTCChromeFactory(WebRTCFactory):
|
||||
"""Sets up the Chrome OS build."""
|
||||
"""Sets up the Chrome Browser+WebRTC build."""
|
||||
|
||||
def __init__(self, build_status_oracle,
|
||||
gclient_solution_name,
|
||||
@ -507,13 +511,48 @@ class WebRTCChromeFactory(WebRTCFactory):
|
||||
svn_url=svn_url,
|
||||
custom_deps_list=custom_deps_list,
|
||||
safesync_url=safesync_url)
|
||||
self.build_enabled = False
|
||||
|
||||
def EnableBuild(self):
|
||||
def EnableBuild(self, release=False, enable_profiling=False):
|
||||
self.AddCommonStep(['rm', '-rf', 'src'], workdir=WEBRTC_BUILD_DIR,
|
||||
descriptor='Cleanup')
|
||||
self.AddGclientSyncStep()
|
||||
if enable_profiling:
|
||||
self.AddCommonStep(['./build/gyp_chromium', '-Dprofiling=1'],
|
||||
descriptor="gyp_chromium",
|
||||
warn_on_failure=True, workdir='build/src')
|
||||
|
||||
if release:
|
||||
self.AddCommonMakeStep('chrome', 'BUILDTYPE=Release')
|
||||
else:
|
||||
self.AddCommonMakeStep('chrome')
|
||||
|
||||
self.build_enabled = True
|
||||
self.release = release
|
||||
self.profiling = enable_profiling
|
||||
|
||||
def EnableBloatCalculation(self):
|
||||
"""Runs a bloat calculation, which will yield a size breakdown for Chrome.
|
||||
|
||||
If running in Release mode, you should also run with profiling to get the
|
||||
symbols right. Running this on Debug mode will work but it will probably
|
||||
take hours.
|
||||
"""
|
||||
assert self.build_enabled is True
|
||||
assert (self.release and self.profiling) or not self.release
|
||||
|
||||
bloat_path = PosixPathJoin(WEBRTC_BUILD_DIR, '..', '..', '..', '..', '..',
|
||||
'..', 'build_internal', 'symsrc',
|
||||
'calculate_bloat.py')
|
||||
output_filename = PosixPathJoin(DEFAULT_BLOAT_DIR, 'bloat_latest.json')
|
||||
build_directory = 'Release' if self.release else 'Debug'
|
||||
chrome_binary = PosixPathJoin('out', build_directory, 'chrome')
|
||||
self.AddCommonStep([bloat_path, '--binary', chrome_binary,
|
||||
'--source-path', '.', '--output-file', output_filename],
|
||||
descriptor='calculate_bloat.py',
|
||||
warn_on_failure=True, workdir='build/src',
|
||||
timeout=7200)
|
||||
|
||||
def AddCommonMakeStep(self, target, make_extra=None):
|
||||
descriptor = ['make ' + target]
|
||||
cmd = ['make', target, '-j100']
|
||||
|
88
tools/continuous_build/build_internal/symsrc/calculate_bloat.py
Executable file
88
tools/continuous_build/build_internal/symsrc/calculate_bloat.py
Executable file
@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python
|
||||
#-*- coding: utf-8 -*-
|
||||
# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
|
||||
#
|
||||
# Use of this source code is governed by a BSD-style license
|
||||
# that can be found in the LICENSE file in the root of the source
|
||||
# tree. An additional intellectual property rights grant can be found
|
||||
# in the file PATENTS. All contributing project authors may
|
||||
# be found in the AUTHORS file in the root of the source tree.
|
||||
|
||||
__author__ = 'phoglund@webrtc.org (Patrik Höglund)'
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
from optparse import OptionParser
|
||||
|
||||
"""Computes a webtreemap-compatible bloat .json file for a binary.
|
||||
|
||||
This will produce an overview of the binary which shows the sizes of its
|
||||
constituent parts. The binary should be built with -g for symbols. If building
|
||||
Chrome, you must include profiling=1 if building in Release mode.
|
||||
|
||||
This script only runs on Linux. It requires the nm utility (part of the binutils
|
||||
package) as well as the bloat.py script. It can run from any working directory.
|
||||
"""
|
||||
|
||||
THIS_SCRIPTS_PATH = os.path.dirname(os.path.realpath(__file__))
|
||||
BLOAT_SCRIPT = THIS_SCRIPTS_PATH + '/../../../third_party/bloat/bloat.py'
|
||||
|
||||
|
||||
def _run_nm(binary):
|
||||
raw_nm_filename = 'nm.out'
|
||||
raw_nm_file = open(raw_nm_filename, 'w')
|
||||
subprocess.check_call(['nm', '-C', '-S', '-l', binary], stdout=raw_nm_file)
|
||||
raw_nm_file.close()
|
||||
return raw_nm_filename
|
||||
|
||||
|
||||
def _run_bloat(raw_nm_filename, source_path, output_filename):
|
||||
json_file = open(output_filename, 'w')
|
||||
subprocess.check_call([BLOAT_SCRIPT,
|
||||
'--strip-prefix=%s' % source_path,
|
||||
'--nm-output=%s' % raw_nm_filename,
|
||||
'syms'], stdout=json_file, stderr=None)
|
||||
json_file.close()
|
||||
|
||||
|
||||
def main():
|
||||
if not os.path.exists(BLOAT_SCRIPT):
|
||||
return 'Missing required dependency bloat (looked in %s).' % BLOAT_SCRIPT
|
||||
|
||||
usage = 'usage: %prog -b <binary> -s <path to source> -o <output JSON file>'
|
||||
parser = OptionParser(usage)
|
||||
parser.add_option('-b', '--binary', dest='binary', default=False,
|
||||
help='Binary to run the bloat calculation on. ' +
|
||||
'The binary should be built with -g for symbols.')
|
||||
parser.add_option('-s', '--source-path', dest='source_path', default=False,
|
||||
help='Where the binary\'s source code is.')
|
||||
parser.add_option('-o', '--output-file', dest='output_file', default=False,
|
||||
help='Where to put the resulting JSON file.')
|
||||
options, unused_args = parser.parse_args()
|
||||
|
||||
if not options.binary:
|
||||
return '%s\n\nYou must specify the binary to run on.' % usage
|
||||
if not options.output_file:
|
||||
return '%s\n\nYou must specify where to put the output file.' % usage
|
||||
if not options.source_path:
|
||||
return '%s\n\nYou must specify the binary\'s source code path.' % usage
|
||||
if not os.path.exists(options.binary):
|
||||
return 'Binary %s does not exist.' % options.binary
|
||||
if not os.path.exists(options.source_path):
|
||||
return 'Source path %s does not exist.' % options.source_path
|
||||
|
||||
# Convert the source path to an absolute path. The ending slash is important
|
||||
# for --strip-prefix later!
|
||||
options.source_path = os.path.realpath(options.source_path) + '/'
|
||||
|
||||
raw_nm_filename = _run_nm(options.binary)
|
||||
_run_bloat(raw_nm_filename, options.source_path, options.output_file)
|
||||
|
||||
os.remove(raw_nm_filename)
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
@ -0,0 +1,44 @@
|
||||
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
|
||||
"http://www.w3.org/TR/html4/strict.dtd">
|
||||
<html>
|
||||
<!--
|
||||
Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
|
||||
|
||||
Use of this source code is governed by a BSD-style license
|
||||
that can be found in the LICENSE file in the root of the source
|
||||
tree. An additional intellectual property rights grant can be found
|
||||
in the file PATENTS. All contributing project authors may
|
||||
be found in the AUTHORS file in the root of the source tree.
|
||||
|
||||
This HTML file displays the WebRTC bloat calculation. Some code here
|
||||
is based on the webtreemap demo file. Copy this file to wherever
|
||||
calculate_bloat.py writes its output .json file (which should be named
|
||||
bloat_latest.json) and it will automatically display the .json file.
|
||||
-->
|
||||
<head>
|
||||
<title>WebRTC Binary Size Tracker</title>
|
||||
<link rel="stylesheet" href="webtreemap.css">
|
||||
<script src="webtreemap.js" type="text/javascript"></script>
|
||||
<script src="bloat_latest.json" type="text/javascript"></script>
|
||||
<style type="text/css">
|
||||
#map {
|
||||
width: 500px;
|
||||
height: 300px;
|
||||
|
||||
position: relative;
|
||||
cursor: pointer;
|
||||
-webkit-user-select: none;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>WebRTC Binary Size Tracker</h1>
|
||||
<p>Click on a box to zoom in. Click on the outermost box to zoom out.</p>
|
||||
<div id="map"></div>
|
||||
<script type="text/javascript">
|
||||
// The kTree variable is defined by bloat_latest.json.
|
||||
var map = document.getElementById("map");
|
||||
appendTreemap(map, kTree);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
Loading…
Reference in New Issue
Block a user