8d3ad82d30
The previous script only looked up the LKGR and generated a commit message. This CL renames the script and makes it update the DEPS file, commit locally, upload CL and send tryjobs. BUG=4688 R=phoglund@webrtc.org Review URL: https://webrtc-codereview.appspot.com/50079004 Cr-Commit-Position: refs/heads/master@{#9284}
392 lines
13 KiB
Python
Executable File
392 lines
13 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
|
|
#
|
|
# Use of this source code is governed by a BSD-style license
|
|
# that can be found in the LICENSE file in the root of the source
|
|
# tree. An additional intellectual property rights grant can be found
|
|
# in the file PATENTS. All contributing project authors may
|
|
# be found in the AUTHORS file in the root of the source tree.
|
|
|
|
"""Script to roll chromium_revision in the WebRTC DEPS file."""
|
|
|
|
import argparse
|
|
import base64
|
|
import collections
|
|
import logging
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import urllib
|
|
|
|
|
|
CHROMIUM_LKGR_URL = 'https://chromium-status.appspot.com/lkgr'
|
|
CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src'
|
|
CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s'
|
|
CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s'
|
|
|
|
COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$')
|
|
CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION=(\d+)$')
|
|
ROLL_BRANCH_NAME = 'roll_chromium_revision'
|
|
|
|
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
|
|
os.pardir))
|
|
sys.path.append(CHECKOUT_ROOT_DIR)
|
|
import setup_links
|
|
|
|
sys.path.append(os.path.join(CHECKOUT_ROOT_DIR, 'tools'))
|
|
import find_depot_tools
|
|
find_depot_tools.add_depot_tools_to_path()
|
|
from gclient import GClientKeywords
|
|
|
|
CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.sh'
|
|
CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join('tools', 'clang', 'scripts',
|
|
'update.sh')
|
|
|
|
DepsEntry = collections.namedtuple('DepsEntry', 'path url revision')
|
|
ChangedDep = collections.namedtuple('ChangedDep', 'path current_rev new_rev')
|
|
|
|
|
|
def ParseDepsDict(deps_content):
|
|
local_scope = {}
|
|
var = GClientKeywords.VarImpl({}, local_scope)
|
|
global_scope = {
|
|
'File': GClientKeywords.FileImpl,
|
|
'From': GClientKeywords.FromImpl,
|
|
'Var': var.Lookup,
|
|
'deps_os': {},
|
|
}
|
|
exec(deps_content, global_scope, local_scope)
|
|
return local_scope
|
|
|
|
|
|
def ParseLocalDepsFile(filename):
|
|
with open(filename, 'rb') as f:
|
|
deps_content = f.read()
|
|
return ParseDepsDict(deps_content)
|
|
|
|
|
|
def ParseRemoteCrDepsFile(revision):
|
|
deps_content = ReadRemoteCrFile('DEPS', revision)
|
|
return ParseDepsDict(deps_content)
|
|
|
|
|
|
def ParseCommitPosition(commit_message):
|
|
for line in reversed(commit_message.splitlines()):
|
|
m = COMMIT_POSITION_RE.match(line.strip())
|
|
if m:
|
|
return m.group(1)
|
|
logging.error('Failed to parse commit position id from:\n%s\n',
|
|
commit_message)
|
|
sys.exit(-1)
|
|
|
|
|
|
def _RunCommand(command, working_dir=None, ignore_exit_code=False,
|
|
extra_env=None):
|
|
"""Runs a command and returns the output from that command.
|
|
|
|
If the command fails (exit code != 0), the function will exit the process.
|
|
|
|
Returns:
|
|
A tuple containing the stdout and stderr outputs as strings.
|
|
"""
|
|
working_dir = working_dir or CHECKOUT_ROOT_DIR
|
|
logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir)
|
|
env = os.environ.copy()
|
|
if extra_env:
|
|
assert all(type(value) == str for value in extra_env.values())
|
|
logging.debug('extra env: %s', extra_env)
|
|
env.update(extra_env)
|
|
p = subprocess.Popen(command, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, env=env,
|
|
cwd=working_dir, universal_newlines=True)
|
|
std_output = p.stdout.read()
|
|
err_output = p.stderr.read()
|
|
p.wait()
|
|
p.stdout.close()
|
|
p.stderr.close()
|
|
if not ignore_exit_code and p.returncode != 0:
|
|
logging.error('Command failed: %s\n'
|
|
'stdout:\n%s\n'
|
|
'stderr:\n%s\n', ' '.join(command), std_output, err_output)
|
|
sys.exit(p.returncode)
|
|
return std_output, err_output
|
|
|
|
|
|
def _GetBranches():
|
|
"""Returns a tuple of active,branches.
|
|
|
|
The 'active' is the name of the currently active branch and 'branches' is a
|
|
list of all branches.
|
|
"""
|
|
lines = _RunCommand(['git', 'branch'])[0].split('\n')
|
|
branches = []
|
|
active = ''
|
|
for line in lines:
|
|
if '*' in line:
|
|
# The assumption is that the first char will always be the '*'.
|
|
active = line[1:].strip()
|
|
branches.append(active)
|
|
else:
|
|
branch = line.strip()
|
|
if branch:
|
|
branches.append(branch)
|
|
return active, branches
|
|
|
|
|
|
def _ReadGitilesContent(url):
|
|
# Download and decode BASE64 content until
|
|
# https://code.google.com/p/gitiles/issues/detail?id=7 is fixed.
|
|
base64_content = ReadUrlContent(url + '?format=TEXT')
|
|
return base64.b64decode(base64_content[0])
|
|
|
|
|
|
def ReadRemoteCrFile(path_below_src, revision):
|
|
"""Reads a remote Chromium file of a specific revision. Returns a string."""
|
|
return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % (revision,
|
|
path_below_src))
|
|
|
|
|
|
def ReadRemoteCrCommit(revision):
|
|
"""Reads a remote Chromium commit message. Returns a string."""
|
|
return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision)
|
|
|
|
|
|
def ReadUrlContent(url):
|
|
"""Connect to a remote host and read the contents. Returns a list of lines."""
|
|
conn = urllib.urlopen(url)
|
|
try:
|
|
return conn.readlines()
|
|
except IOError as e:
|
|
logging.exception('Error connecting to %s. Error: %s', url, e)
|
|
raise
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def GetMatchingDepsEntries(depsentry_dict, dir_path):
|
|
"""Gets all deps entries matching the provided path.
|
|
|
|
This list may contain more than one DepsEntry object.
|
|
Example: dir_path='src/testing' would give results containing both
|
|
'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's DEPS.
|
|
Example 2: dir_path='src/build' should return 'src/build' but not
|
|
'src/buildtools'.
|
|
|
|
Returns:
|
|
A list of DepsEntry objects.
|
|
"""
|
|
result = []
|
|
for path, depsentry in depsentry_dict.iteritems():
|
|
if path == dir_path:
|
|
result.append(depsentry)
|
|
else:
|
|
parts = path.split(os.sep)
|
|
if all(part == parts[i]
|
|
for i, part in enumerate(dir_path.split(os.sep))):
|
|
result.append(depsentry)
|
|
return result
|
|
|
|
|
|
def BuildDepsentryDict(deps_dict):
|
|
"""Builds a dict of DepsEntry object from a raw parsed deps dict."""
|
|
result = {}
|
|
def AddDepsEntries(deps_subdict):
|
|
for path, deps_url in deps_subdict.iteritems():
|
|
if not result.has_key(path):
|
|
url, revision = deps_url.split('@') if deps_url else (None, None)
|
|
result[path] = DepsEntry(path, url, revision)
|
|
|
|
AddDepsEntries(deps_dict['deps'])
|
|
for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']:
|
|
AddDepsEntries(deps_dict['deps_os'].get(deps_os, {}))
|
|
return result
|
|
|
|
|
|
def CalculateChangedDeps(current_deps, new_deps):
|
|
result = []
|
|
current_entries = BuildDepsentryDict(current_deps)
|
|
new_entries = BuildDepsentryDict(new_deps)
|
|
|
|
all_deps_dirs = setup_links.DIRECTORIES
|
|
for deps_dir in all_deps_dirs:
|
|
# All deps have 'src' prepended to the path in the Chromium DEPS file.
|
|
dir_path = 'src/%s' % deps_dir
|
|
|
|
for entry in GetMatchingDepsEntries(current_entries, dir_path):
|
|
new_matching_entries = GetMatchingDepsEntries(new_entries, entry.path)
|
|
assert len(new_matching_entries) <= 1, (
|
|
'Should never find more than one entry matching %s in %s, found %d' %
|
|
(entry.path, new_entries, len(new_matching_entries)))
|
|
if not new_matching_entries:
|
|
result.append(ChangedDep(entry.path, entry.revision, 'None'))
|
|
elif entry != new_matching_entries[0]:
|
|
result.append(ChangedDep(entry.path, entry.revision,
|
|
new_matching_entries[0].revision))
|
|
return result
|
|
|
|
|
|
def CalculateChangedClang(new_cr_rev):
|
|
def GetClangRev(lines):
|
|
for line in lines:
|
|
match = CLANG_REVISION_RE.match(line)
|
|
if match:
|
|
return match.group(1)
|
|
return None
|
|
|
|
chromium_src_path = os.path.join(CHECKOUT_ROOT_DIR, 'chromium', 'src',
|
|
CLANG_UPDATE_SCRIPT_LOCAL_PATH)
|
|
with open(chromium_src_path, 'rb') as f:
|
|
current_lines = f.readlines()
|
|
current_rev = GetClangRev(current_lines)
|
|
|
|
new_clang_update_sh = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH,
|
|
new_cr_rev).splitlines()
|
|
new_rev = GetClangRev(new_clang_update_sh)
|
|
return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, current_rev, new_rev)
|
|
|
|
|
|
def GenerateCommitMessage(current_cr_rev, new_cr_rev, changed_deps_list,
|
|
clang_change):
|
|
current_cr_rev = current_cr_rev[0:7]
|
|
new_cr_rev = new_cr_rev[0:7]
|
|
rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev)
|
|
|
|
current_git_number = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev))
|
|
new_git_number = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev))
|
|
git_number_interval = '%s:%s' % (current_git_number, new_git_number)
|
|
|
|
commit_msg = ['Roll chromium_revision %s (%s)' % (rev_interval,
|
|
git_number_interval)]
|
|
|
|
if changed_deps_list:
|
|
commit_msg.append('\nRelevant changes:')
|
|
|
|
for c in changed_deps_list:
|
|
commit_msg.append('* %s: %s..%s' % (c.path, c.current_rev[0:7],
|
|
c.new_rev[0:7]))
|
|
|
|
change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS')
|
|
commit_msg.append('Details: %s' % change_url)
|
|
|
|
if clang_change.current_rev != clang_change.new_rev:
|
|
commit_msg.append('\nClang version changed %s:%s' %
|
|
(clang_change.current_rev, clang_change.new_rev))
|
|
change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval,
|
|
CLANG_UPDATE_SCRIPT_URL_PATH)
|
|
commit_msg.append('Details: %s' % change_url)
|
|
else:
|
|
commit_msg.append('\nClang version was not updated in this roll.')
|
|
return '\n'.join(commit_msg)
|
|
|
|
|
|
def UpdateDeps(deps_filename, old_cr_revision, new_cr_revision):
|
|
"""Update the DEPS file with the new revision."""
|
|
with open(deps_filename, 'rb') as deps_file:
|
|
deps_content = deps_file.read()
|
|
deps_content = deps_content.replace(old_cr_revision, new_cr_revision)
|
|
with open(deps_filename, 'wb') as deps_file:
|
|
deps_file.write(deps_content)
|
|
|
|
|
|
def _CreateRollBranch(dry_run):
|
|
current_branch = _RunCommand(
|
|
['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[0].splitlines()[0]
|
|
if current_branch != 'master':
|
|
logging.error('Please checkout the master branch and re-run this script.')
|
|
if not dry_run:
|
|
sys.exit(-1)
|
|
|
|
logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME)
|
|
if not dry_run:
|
|
_RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME])
|
|
|
|
|
|
def _RemovePreviousRollBranch(dry_run):
|
|
active_branch, branches = _GetBranches()
|
|
if active_branch == ROLL_BRANCH_NAME:
|
|
active_branch = 'master'
|
|
if ROLL_BRANCH_NAME in branches:
|
|
logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME)
|
|
if not dry_run:
|
|
_RunCommand(['git', 'checkout', active_branch])
|
|
_RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
|
|
|
|
|
|
def _LocalCommit(commit_msg, dry_run):
|
|
logging.info('Committing changes locally.')
|
|
if not dry_run:
|
|
_RunCommand(['git', 'add', '--update', '.'])
|
|
_RunCommand(['git', 'commit', '-m', commit_msg])
|
|
|
|
|
|
def _UploadCL(dry_run):
|
|
logging.info('Uploading CL...')
|
|
if not dry_run:
|
|
_RunCommand(['git', 'cl', 'upload'], extra_env={'EDITOR': 'true'})
|
|
|
|
|
|
def _LaunchTrybots(dry_run):
|
|
logging.info('Sending tryjobs...')
|
|
if not dry_run:
|
|
_RunCommand(['git', 'cl', 'try'])
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('--clean', action='store_true', default=False,
|
|
help='Removes any previous local roll branch.')
|
|
p.add_argument('-r', '--revision',
|
|
help=('Chromium Git revision to roll to. Defaults to the '
|
|
'Chromium LKGR revision if omitted.'))
|
|
p.add_argument('--dry-run', action='store_true', default=False,
|
|
help=('Calculate changes and modify DEPS, but don\'t create '
|
|
'any local branch, commit, upload CL or send any '
|
|
'tryjobs.'))
|
|
p.add_argument('-v', '--verbose', action='store_true', default=False,
|
|
help='Be extra verbose in printing of log messages.')
|
|
opts = p.parse_args()
|
|
|
|
if opts.verbose:
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
else:
|
|
logging.basicConfig(level=logging.INFO)
|
|
|
|
if opts.clean:
|
|
_RemovePreviousRollBranch(opts.dry_run)
|
|
|
|
if not opts.revision:
|
|
lkgr_contents = ReadUrlContent(CHROMIUM_LKGR_URL)
|
|
logging.info('No revision specified. Using LKGR: %s', lkgr_contents[0])
|
|
opts.revision = lkgr_contents[0]
|
|
|
|
deps_filename = os.path.join(CHECKOUT_ROOT_DIR, 'DEPS')
|
|
local_deps = ParseLocalDepsFile(deps_filename)
|
|
current_cr_rev = local_deps['vars']['chromium_revision']
|
|
|
|
current_cr_deps = ParseRemoteCrDepsFile(current_cr_rev)
|
|
new_cr_deps = ParseRemoteCrDepsFile(opts.revision)
|
|
|
|
changed_deps = sorted(CalculateChangedDeps(current_cr_deps, new_cr_deps))
|
|
clang_change = CalculateChangedClang(opts.revision)
|
|
if changed_deps or clang_change:
|
|
commit_msg = GenerateCommitMessage(current_cr_rev, opts.revision,
|
|
changed_deps, clang_change)
|
|
logging.debug('Commit message:\n%s', commit_msg)
|
|
else:
|
|
logging.info('No deps changes detected when rolling from %s to %s. '
|
|
'Aborting without action.', current_cr_rev, opts.revision)
|
|
return 0
|
|
|
|
_CreateRollBranch(opts.dry_run)
|
|
UpdateDeps(deps_filename, current_cr_rev, opts.revision)
|
|
_LocalCommit(commit_msg, opts.dry_run)
|
|
_UploadCL(opts.dry_run)
|
|
_LaunchTrybots(opts.dry_run)
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|