From f1e020ec9973b0eb139e1850f5d791e0129d6114 Mon Sep 17 00:00:00 2001 From: "kjellander@webrtc.org" Date: Thu, 12 Apr 2012 06:18:10 +0000 Subject: [PATCH] Cleanup script using Python instead of rm command. BUG=None TEST=Ran the script on all platforms and a Linux master+slave build. Review URL: https://webrtc-codereview.appspot.com/479008 git-svn-id: http://webrtc.googlecode.com/svn/trunk@2010 4adac7df-926f-26a2-2b94-8c16560cd09d --- tools/.gitignore | 1 + .../scripts/webrtc_buildbot/utils.py | 48 +++++++++++++----- .../site_config/config_private.py | 1 + .../build_internal/symsrc/cleanup_build.py | 50 +++++++++++++++++++ 4 files changed, 87 insertions(+), 13 deletions(-) create mode 100755 tools/continuous_build/build_internal/symsrc/cleanup_build.py diff --git a/tools/.gitignore b/tools/.gitignore index 6aaedbeb7..b56099f23 100644 --- a/tools/.gitignore +++ b/tools/.gitignore @@ -39,6 +39,7 @@ robots.txt # Buildbot slave log directories: /continuous_build/build_internal/masters/master.webrtc/android +/continuous_build/build_internal/masters/master.webrtc/android-ndk /continuous_build/build_internal/masters/master.webrtc/chromeos /continuous_build/build_internal/masters/master.webrtc/linux-chrome /continuous_build/build_internal/masters/master.webrtc/linux-clang diff --git a/tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py b/tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py index bad8d74d1..bcd5e0ede 100755 --- a/tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py +++ b/tools/continuous_build/build_internal/scripts/webrtc_buildbot/utils.py @@ -10,7 +10,9 @@ __author__ = 'ivinnichenko@webrtc.org (Illya Vinnichenko)' import buildbot +import ntpath import os +import posixpath import sys import urlparse from buildbot.process import factory @@ -90,6 +92,7 @@ class WebRTCFactory(factory.BuildFactory): self.properties = properties.Properties() self.gyp_params = [] self.release = False + self.path_joiner = PosixPathJoin def EnableBuild(self): """Adds steps for building WebRTC [must be overridden]. @@ -164,7 +167,7 @@ class WebRTCFactory(factory.BuildFactory): thinks the last build failed. Otherwise it cleans just the build output. """ self.addStep(SmartClean(self.build_status_oracle, self.is_try_slave, - workdir=WEBRTC_BUILD_DIR)) + self.path_joiner, workdir=WEBRTC_BUILD_DIR)) def AddCommonTestRunStep(self, test, descriptor='', cmd=None): """Adds a step for running a single test [must be overridden]. @@ -312,12 +315,12 @@ class BuildStatusOracle: self.builder_name = builder_name self.master_work_dir = DEFAULT_MASTER_WORK_DIR - def LastBuildFailed(self): + def LastBuildSucceeded(self): failure_file_path = self._GetFailureBuildPath() - return os.path.exists(failure_file_path) + return not os.path.exists(failure_file_path) def ForgetLastBuild(self): - if self.LastBuildFailed(): + if not self.LastBuildSucceeded(): os.remove(self._GetFailureBuildPath()) def SetLastBuildAsFailed(self): @@ -344,28 +347,37 @@ class MonitoredShellCommand(ShellCommand): class SmartClean(ShellCommand): """Cleans the repository fully or partially depending on the build state.""" - def __init__(self, build_status_oracle, is_try_slave, **kwargs): + + def __init__(self, build_status_oracle, is_try_slave, path_joiner, **kwargs): + """Args: + build_status_oracle: class that knows if the previous build failed. + is_try_slave: if the current factory is a try slave. + path_joiner: function to create paths for the current platform, given + a number of path elements in string form. + """ ShellCommand.__init__(self, **kwargs) self.addFactoryArguments(build_status_oracle=build_status_oracle, - is_try_slave=is_try_slave) + is_try_slave=is_try_slave, path_joiner=path_joiner) + self.name = "Clean" self.haltOnFailure = True self.build_status_oracle = build_status_oracle self.is_try_slave = is_try_slave + self.clean_script = path_joiner(WEBRTC_BUILD_DIR, '..', '..', '..', '..', + '..', 'build_internal', 'symsrc', + 'cleanup_build.py') def start(self): # Always do normal clean for try slaves, since nuking confuses the Chromium # scripts' GClient sync step. - if self.is_try_slave or self.build_status_oracle.LastBuildFailed(): + if self.is_try_slave or self.build_status_oracle.LastBuildSucceeded(): self.description = ['Clean'] - self.setCommand('rm -rf trunk/out && ' - 'rm -rf trunk/xcodebuild &&' - 'rm -rf trunk/build/Debug &&' - 'rm -rf trunk/build/Release') + cmd = 'python %s ' % self.clean_script else: self.build_status_oracle.ForgetLastBuild() self.description = ['Nuke Repository', '(Previous Failed)'] - self.setCommand(['rm', '-rf', 'trunk']) + cmd = 'python %s --nuke' % self.clean_script + self.setCommand(cmd) ShellCommand.start(self) @@ -749,6 +761,7 @@ class WebRTCWinFactory(WebRTCFactory): self.platform = 'x64' self.allowed_platforms = ['x64', 'Win32'] self.allowed_configurations = ['Debug', 'Release', 'both'] + self.path_joiner = WindowsPathJoin def AddCommonStep(self, cmd, descriptor='', workdir=WEBRTC_TRUNK_DIR, halt_build_on_failure=True, warn_on_failure=False): @@ -787,7 +800,9 @@ class WebRTCWinFactory(WebRTCFactory): # To avoid having to modify kill_processes.py, we set the working dir to # the build dir (three levels up from the build dir that contains # third_party/psutils). - cmd = 'python ..\\..\\..\\scripts\\slave\\kill_processes.py' + kill_script = self.PathJoin(WEBRTC_BUILD_DIR, '..', '..', '..', 'scripts', + 'slave', 'kill_processes.py') + cmd = 'python %s' % kill_script self.AddCommonStep(cmd, 'taskkill', workdir=WEBRTC_BUILD_DIR) # Now do the clean + build. @@ -827,8 +842,15 @@ class WebRTCWinFactory(WebRTCFactory): self.AddCommonStep(cmd, descriptor=descriptor, halt_build_on_failure=False) + # Utility functions +def PosixPathJoin(*args): + return posixpath.normpath(posixpath.join(*args)) + +def WindowsPathJoin(*args): + return ntpath.normpath(ntpath.join(*args)) + class UnsupportedConfigurationError(Exception): pass diff --git a/tools/continuous_build/build_internal/site_config/config_private.py b/tools/continuous_build/build_internal/site_config/config_private.py index c7d366e23..04010c7b1 100644 --- a/tools/continuous_build/build_internal/site_config/config_private.py +++ b/tools/continuous_build/build_internal/site_config/config_private.py @@ -24,6 +24,7 @@ class Master(object): # Directly fetches from anonymous webkit svn server. webkit_root_url = 'http://svn.webkit.org/repository/webkit' + nacl_trunk_url = 'http://src.chromium.org/native_client/trunk' llvm_url = 'http://llvm.org/svn/llvm-project' diff --git a/tools/continuous_build/build_internal/symsrc/cleanup_build.py b/tools/continuous_build/build_internal/symsrc/cleanup_build.py new file mode 100755 index 000000000..5c6ab2f42 --- /dev/null +++ b/tools/continuous_build/build_internal/symsrc/cleanup_build.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +__author__ = 'kjellander@webrtc.org (Henrik Kjellander)' + +"""Clean up a build by removing just compile output or everything (nuke). + +This script assumes the trunk directory is located directly below the current +working directory. +""" + +from optparse import OptionParser +import sys + +from common import chromium_utils + + +def main(): + usage = 'usage: %prog [--nuke]' + parser = OptionParser(usage) + parser.add_option('-n', '--nuke', + action='store_true', dest='nuke', default=False, + help='Nuke whole repository (not just build output)') + options, unused_args = parser.parse_args() + + if options.nuke: + chromium_utils.RemoveDirectory('trunk') + else: + # Remove platform specific build output directories. + if chromium_utils.IsWindows(): + chromium_utils.RemoveDirectory('trunk\\build\\Debug') + chromium_utils.RemoveDirectory('trunk\\build\\Release') + elif chromium_utils.IsMac(): + chromium_utils.RemoveDirectory('trunk/out') + chromium_utils.RemoveDirectory('trunk/xcodebuild') + elif chromium_utils.IsLinux(): + chromium_utils.RemoveDirectory('trunk/out') + else: + print 'Unknown platform: ' + sys.platform + return 1 + return 0 + +if '__main__' == __name__: + sys.exit(main())