Implemented bloat calculation. This will measure the binary size of Chrome+WebRTC components each weekend.

BUG=
TEST=

Review URL: https://webrtc-codereview.appspot.com/508005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@2088 4adac7df-926f-26a2-2b94-8c16560cd09d

tools/DEPS

@@ -25,7 +25,13 @@ deps = {
 
   # Used by tools/quality_tracking/dashboard and tools/python_charts.
   "tools/third_party/google-visualization-python":
-     "http://google-visualization-python.googlecode.com/svn/trunk/@15",
+    "http://google-visualization-python.googlecode.com/svn/trunk/@15",
+
+  # Used by tools/continuous_build/build_internal/symsrc/calculate_bloat.py.
+  "tools/third_party/bloat":
+    "https://github.com/martine/bloat.git@31428aaa491",
+  "tools/third_party/webtreemap":
+    "https://github.com/martine/webtreemap.git@7839cf9154",
 }
 
 hooks = [

tools/continuous_build/build_internal/masters/master.webrtc/master.cfg

@@ -17,6 +17,7 @@ from buildbot.changes.pb import PBChangeSource
 from buildbot.changes.svnpoller import SVNPoller
 from buildbot.process import factory
 from buildbot.scheduler import Scheduler
+from buildbot.schedulers import timed
 from buildbot.status import html
 from buildbot.status import mail
 from buildbot.steps import shell
@@ -54,10 +55,20 @@ webrtc_scheduler = Scheduler(name='all', branch=None, treeStableTimer=5*60,
                                            'AndroidNDK',
                                            'ChromeOS'
                                            ])
+
 chrome_scheduler = Scheduler(name='chrome', branch=None, treeStableTimer=60*60,
                              builderNames=['Chrome'])
 
-c['schedulers'] = [webrtc_scheduler, chrome_scheduler]
+# Run the weekend scheduler at sunday, 2 AM CST/CDT. This will mean roughly
+# Sunday 9 AM in the CET timezone, which should avoid everyone's working hours.
+weekend_scheduler = timed.Nightly(name='weekend',
+                                  builderNames=['ChromeBloat'],
+                                  branch=None,
+                                  dayOfWeek=6,
+                                  hour=2,
+                                  minute=0)
+
+c['schedulers'] = [webrtc_scheduler, chrome_scheduler, weekend_scheduler]
 
 ####### TESTS
 # Defines the supported tests followed by a tuple defining if the tests are
@@ -163,21 +174,33 @@ chromeos_factory = utils.WebRTCLinuxFactory(
 chromeos_factory.EnableBuild(chrome_os=True)
 chromeos_factory.EnableTests(linux_normal_tests)
 
+CHROME_SVN_URL = 'http://src.chromium.org/svn/trunk/src'
+CHROME_LKGR_URL = 'http://chromium-status.appspot.com/lkgr'
+CHROME_GCLIENT_SOLUTION_NAME='src'
+CHROME_CUSTOM_DEPS_LIST = [
+  ('src/third_party/webrtc', 'http://webrtc.googlecode.com/svn/stable/src'),
+  ('src/third_party/WebKit/LayoutTests', None),
+  ('src/chrome/tools/test/reference_build', None),
+]
+
 linux_chrome_factory = utils.WebRTCChromeFactory(
     utils.BuildStatusOracle('linux_chrome'),
-    gclient_solution_name='src',
-    svn_url='http://src.chromium.org/svn/trunk/src',
-    custom_deps_list=[
-        ('src/third_party/webrtc',
-         'http://webrtc.googlecode.com/svn/stable/src'),
-        ('src/third_party/WebKit/LayoutTests',
-         None),
-        ('src/chrome/tools/test/reference_build',
-         None),
-    ],
-    safesync_url='http://chromium-status.appspot.com/lkgr')
+    gclient_solution_name=CHROME_GCLIENT_SOLUTION_NAME,
+    svn_url=CHROME_SVN_URL,
+    custom_deps_list=CHROME_CUSTOM_DEPS_LIST,
+    safesync_url=CHROME_LKGR_URL)
 linux_chrome_factory.EnableBuild()
 
+linux_chrome_bloat_factory = utils.WebRTCChromeFactory(
+    utils.BuildStatusOracle('linux_chrome_bloat'),
+    gclient_solution_name=CHROME_GCLIENT_SOLUTION_NAME,
+    svn_url=CHROME_SVN_URL,
+    custom_deps_list=CHROME_CUSTOM_DEPS_LIST,
+    safesync_url=CHROME_LKGR_URL)
+linux_chrome_bloat_factory.EnableBuild(release=True, enable_profiling=True)
+linux_chrome_bloat_factory.EnableBloatCalculation()
+
+
 linux_clang = utils.WebRTCLinuxFactory(
     utils.BuildStatusOracle('linux_clang'))
 linux_clang.EnableBuild(clang=True)
@@ -292,6 +315,12 @@ linux_builder_chrome = {
       'builddir': 'linux-chrome',
       'factory': linux_chrome_factory,
       }
+linux_builder_chrome_bloat = {
+      'name': 'ChromeBloat',
+      'slavename': 'webrtc-chrome',
+      'builddir': 'linux-chrome-bloat',
+      'factory': linux_chrome_bloat_factory,
+      }
 linux_builder_clang = {
       'name': 'LinuxClang',
       'slavename': 'webrtc-cb-linux-slave-8',
@@ -339,6 +368,7 @@ c['builders'] = [
       android_builder_ndk,
       chromeos_builder,
       linux_builder_chrome,
+      linux_builder_chrome_bloat,
       ]
 
 

tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py

@@ -35,6 +35,7 @@ WEBRTC_BUILD_DIR = 'build/'
 VALGRIND_CMD = ['tools/valgrind-webrtc/webrtc_tests.sh', '-t', 'cmdline']
 
 DEFAULT_COVERAGE_DIR = '/var/www/coverage/'
+DEFAULT_BLOAT_DIR = '/var/www/bloat/'
 DEFAULT_MASTER_WORK_DIR = '.'
 GCLIENT_RETRIES = 3
 
@@ -130,7 +131,8 @@ class WebRTCFactory(factory.BuildFactory):
       self.EnableTest(test)
 
   def AddCommonStep(self, cmd, descriptor='', workdir=WEBRTC_TRUNK_DIR,
-                    halt_build_on_failure=True, warn_on_failure=False):
+                    halt_build_on_failure=True, warn_on_failure=False,
+                    timeout=1200):
     """Adds a step which will run as a shell command on the slave.
 
        NOTE: you are recommended to use this method to add new shell commands
@@ -155,6 +157,7 @@ class WebRTCFactory(factory.BuildFactory):
              warn_on_failure.
          warn_on_failure: If true, this step isn't that important and will not
              cause a failed build on failure.
+         timeout: The timeout for the command, in seconds.
     """
     flunk_on_failure = not warn_on_failure
 
@@ -173,7 +176,8 @@ class WebRTCFactory(factory.BuildFactory):
         warnOnFailure=warn_on_failure,
         flunkOnFailure=flunk_on_failure,
         haltOnFailure=halt_build_on_failure,
-        name='_'.join(descriptor)))
+        name='_'.join(descriptor),
+        timeout=timeout))
 
   def AddSmartCleanStep(self):
     """Adds a smart clean step.
@@ -230,9 +234,9 @@ class WebRTCFactory(factory.BuildFactory):
     # Try 4+1=5 times, 10 seconds apart.
     retry = (10, 4)
     # Subversion timeout is by default 2 minutes; we allow 5 minutes.
-    timeout = 60*5
+    timeout = 60 * 5
     # Removal can take a long time. Allow 15 minutes.
-    rm_timeout = 60*15
+    rm_timeout = 60 * 15
     self.addStep(chromium_step.GClient,
                  gclient_spec=gclient_spec,
                  svnurl=WEBRTC_SVN_LOCATION,
@@ -495,7 +499,7 @@ class WebRTCAndroidNDKFactory(WebRTCFactory):
     self.AddCommonStep(cmd=full_cmd, descriptor=descriptor)
 
 class WebRTCChromeFactory(WebRTCFactory):
-  """Sets up the Chrome OS build."""
+  """Sets up the Chrome Browser+WebRTC build."""
 
   def __init__(self, build_status_oracle,
                gclient_solution_name,
@@ -507,12 +511,47 @@ class WebRTCChromeFactory(WebRTCFactory):
                            svn_url=svn_url,
                            custom_deps_list=custom_deps_list,
                            safesync_url=safesync_url)
+    self.build_enabled = False
 
-  def EnableBuild(self):
+  def EnableBuild(self, release=False, enable_profiling=False):
     self.AddCommonStep(['rm', '-rf', 'src'], workdir=WEBRTC_BUILD_DIR,
                        descriptor='Cleanup')
     self.AddGclientSyncStep()
-    self.AddCommonMakeStep('chrome')
+    if enable_profiling:
+      self.AddCommonStep(['./build/gyp_chromium', '-Dprofiling=1'],
+                         descriptor="gyp_chromium",
+                         warn_on_failure=True, workdir='build/src')
+
+    if release:
+      self.AddCommonMakeStep('chrome', 'BUILDTYPE=Release')
+    else:
+      self.AddCommonMakeStep('chrome')
+
+    self.build_enabled = True
+    self.release = release
+    self.profiling = enable_profiling
+
+  def EnableBloatCalculation(self):
+    """Runs a bloat calculation, which will yield a size breakdown for Chrome.
+
+    If running in Release mode, you should also run with profiling to get the
+    symbols right. Running this on Debug mode will work but it will probably
+    take hours.
+    """
+    assert self.build_enabled is True
+    assert (self.release and self.profiling) or not self.release
+
+    bloat_path = PosixPathJoin(WEBRTC_BUILD_DIR, '..', '..', '..', '..', '..',
+                               '..', 'build_internal', 'symsrc',
+                               'calculate_bloat.py')
+    output_filename = PosixPathJoin(DEFAULT_BLOAT_DIR, 'bloat_latest.json')
+    build_directory = 'Release' if self.release else 'Debug'
+    chrome_binary = PosixPathJoin('out', build_directory, 'chrome')
+    self.AddCommonStep([bloat_path, '--binary', chrome_binary,
+                        '--source-path', '.', '--output-file', output_filename],
+                       descriptor='calculate_bloat.py',
+                       warn_on_failure=True, workdir='build/src',
+                       timeout=7200)
 
   def AddCommonMakeStep(self, target, make_extra=None):
     descriptor = ['make ' + target]

tools/continuous_build/build_internal/symsrc/calculate_bloat.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+#  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+#
+#  Use of this source code is governed by a BSD-style license
+#  that can be found in the LICENSE file in the root of the source
+#  tree. An additional intellectual property rights grant can be found
+#  in the file PATENTS.  All contributing project authors may
+#  be found in the AUTHORS file in the root of the source tree.
+
+__author__ = 'phoglund@webrtc.org (Patrik Höglund)'
+
+import os
+import sys
+import subprocess
+
+from optparse import OptionParser
+
+"""Computes a webtreemap-compatible bloat .json file for a binary.
+
+This will produce an overview of the binary which shows the sizes of its
+constituent parts. The binary should be built with -g for symbols. If building
+Chrome, you must include profiling=1 if building in Release mode.
+
+This script only runs on Linux. It requires the nm utility (part of the binutils
+package) as well as the bloat.py script. It can run from any working directory.
+"""
+
+THIS_SCRIPTS_PATH = os.path.dirname(os.path.realpath(__file__))
+BLOAT_SCRIPT = THIS_SCRIPTS_PATH + '/../../../third_party/bloat/bloat.py'
+
+
+def _run_nm(binary):
+  raw_nm_filename = 'nm.out'
+  raw_nm_file = open(raw_nm_filename, 'w')
+  subprocess.check_call(['nm', '-C', '-S', '-l', binary], stdout=raw_nm_file)
+  raw_nm_file.close()
+  return raw_nm_filename
+
+
+def _run_bloat(raw_nm_filename, source_path, output_filename):
+  json_file = open(output_filename, 'w')
+  subprocess.check_call([BLOAT_SCRIPT,
+                        '--strip-prefix=%s' % source_path,
+                        '--nm-output=%s' % raw_nm_filename,
+                        'syms'], stdout=json_file, stderr=None)
+  json_file.close()
+
+
+def main():
+  if not os.path.exists(BLOAT_SCRIPT):
+    return 'Missing required dependency bloat (looked in %s).' % BLOAT_SCRIPT
+
+  usage = 'usage: %prog -b <binary> -s <path to source> -o <output JSON file>'
+  parser = OptionParser(usage)
+  parser.add_option('-b', '--binary', dest='binary', default=False,
+                    help='Binary to run the bloat calculation on. ' +
+                         'The binary should be built with -g for symbols.')
+  parser.add_option('-s', '--source-path', dest='source_path', default=False,
+                    help='Where the binary\'s source code is.')
+  parser.add_option('-o', '--output-file', dest='output_file', default=False,
+                    help='Where to put the resulting JSON file.')
+  options, unused_args = parser.parse_args()
+
+  if not options.binary:
+    return '%s\n\nYou must specify the binary to run on.' % usage
+  if not options.output_file:
+    return '%s\n\nYou must specify where to put the output file.' % usage
+  if not options.source_path:
+    return '%s\n\nYou must specify the binary\'s source code path.' % usage
+  if not os.path.exists(options.binary):
+    return 'Binary %s does not exist.' % options.binary
+  if not os.path.exists(options.source_path):
+    return 'Source path %s does not exist.' % options.source_path
+
+  # Convert the source path to an absolute path. The ending slash is important
+  # for --strip-prefix later!
+  options.source_path = os.path.realpath(options.source_path) + '/'
+
+  raw_nm_filename = _run_nm(options.binary)
+  _run_bloat(raw_nm_filename, options.source_path, options.output_file)
+
+  os.remove(raw_nm_filename)
+
+  return 0
+
+if __name__ == '__main__':
+  sys.exit(main())

tools/continuous_build/build_internal/symsrc/webrtc_bloat.html

@@ -0,0 +1,44 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
+    "http://www.w3.org/TR/html4/strict.dtd">
+<html>
+  <!--
+  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+
+  Use of this source code is governed by a BSD-style license
+  that can be found in the LICENSE file in the root of the source
+  tree. An additional intellectual property rights grant can be found
+  in the file PATENTS.  All contributing project authors may
+  be found in the AUTHORS file in the root of the source tree.
+
+  This HTML file displays the WebRTC bloat calculation. Some code here
+  is based on the webtreemap demo file. Copy this file to wherever
+  calculate_bloat.py writes its output .json file (which should be named
+  bloat_latest.json) and it will automatically display the .json file.
+  -->
+  <head>
+    <title>WebRTC Binary Size Tracker</title>
+    <link rel="stylesheet" href="webtreemap.css">
+    <script src="webtreemap.js" type="text/javascript"></script>
+    <script src="bloat_latest.json" type="text/javascript"></script>
+    <style type="text/css">
+      #map {
+        width: 500px;
+        height: 300px;
+
+        position: relative;
+        cursor: pointer;
+       -webkit-user-select: none;
+      }
+    </style>
+  </head>
+  <body>
+    <h1>WebRTC Binary Size Tracker</h1>
+    <p>Click on a box to zoom in. Click on the outermost box to zoom out.</p>
+    <div id="map"></div>
+    <script type="text/javascript">
+      // The kTree variable is defined by bloat_latest.json.
+      var map = document.getElementById("map");
+      appendTreemap(map, kTree);
+    </script>
+  </body>
+</html>