From 7c78d24011696791f48a539260b22cec52d96c38 Mon Sep 17 00:00:00 2001
From: Dan Albert <danalbert@google.com>
Date: Fri, 9 Jan 2015 14:12:52 -0800
Subject: [PATCH] Check in bionicbb code.
These have been sitting around in a git repo on my machine for a
while. They're now big an important enough that I'd like to both keep
them securely backed up, and also have my changes reviewed.
Change-Id: Ic4545149b4b07f0d57b21cac32aab8553dceb567
---
tools/bionicbb/.gitignore | 56 ++++++
tools/bionicbb/README.md | 83 ++++++++
tools/bionicbb/__init__.py | 0
tools/bionicbb/build_listener.py | 104 ++++++++++
tools/bionicbb/gerrit.py | 56 ++++++
tools/bionicbb/gmail_listener.py | 314 +++++++++++++++++++++++++++++++
tools/bionicbb/setup.cfg | 2 +
7 files changed, 615 insertions(+)
create mode 100644 tools/bionicbb/.gitignore
create mode 100644 tools/bionicbb/README.md
create mode 100644 tools/bionicbb/__init__.py
create mode 100644 tools/bionicbb/build_listener.py
create mode 100644 tools/bionicbb/gerrit.py
create mode 100644 tools/bionicbb/gmail_listener.py
create mode 100644 tools/bionicbb/setup.cfg
diff --git a/tools/bionicbb/.gitignore b/tools/bionicbb/.gitignore
new file mode 100644
index 000000000..76ff599eb
--- /dev/null
+++ b/tools/bionicbb/.gitignore
@@ -0,0 +1,56 @@
+config.py
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
diff --git a/tools/bionicbb/README.md b/tools/bionicbb/README.md
new file mode 100644
index 000000000..d2dd2e06c
--- /dev/null
+++ b/tools/bionicbb/README.md
@@ -0,0 +1,83 @@
+bionicbb
+========
+
+The bionic buildbot contains two services: a gmail polling service, and a web
+service that interacts with gerrit.
+
+Dependencies
+------------
+
+ * Python 2.7
+ * [Flask](http://flask.pocoo.org/)
+ * [Google API Client Library](https://developers.google.com/api-client-library/python/start/installation)
+ * [jenkinsapi](https://pypi.python.org/pypi/jenkinsapi)
+ * [Requests](http://docs.python-requests.org/en/latest/)
+ * [termcolor](https://pypi.python.org/pypi/termcolor)
+
+Setup
+-----
+
+Create a `config.py` in the same directory as the sources. The structure of the
+configuration file is as follows:
+
+ client_secret_file = 'CLIENT_SECRET_FILE.json'
+ jenkins_credentials = {
+ 'username': 'JENKINS_USERNAME',
+ 'password': 'JENKINS_PASSWORD',
+ }
+
+The client secret file comes from the Gmail API page of the [Google Developers
+Console](https://console.developers.google.com/). The Jenkins credentials are
+for a Jenkins account that has the appropriate permissions to launch the jobs
+the buildbot will use.
+
+You will also need to add the HTTP password for the buildbot's Gerrit account to
+`~/.netrc`. The HTTP password can be obtained from the [Gerrit HTTP password
+settings](https://android-review.googlesource.com/#/settings/http-password).
+
+To launch the services:
+
+ $ python build_listener.py >build.log 2>&1 &
+ $ python gmail_listener.py >mail.log 2>&1 &
+
+The mail listener will direct your browser to an authentication page for the
+Gmail API.
+
+gmail\_listener.py
+------------------
+
+Bionicbb polls a gmail account to find changes that need to be built. The gmail
+account needs to have a gerrit account set up with project watches on anything
+it finds interesting. This is a rather ugly hack, but it seems to be the
+simplest option available.
+
+Gerrit does offer a streaming notification service that would be _far_ better,
+but it is only available over an SSH conection to gerrit, and the AOSP gerrit
+does not support this connection.
+
+Another option would be polling gerrit itself, but we'd have to process each
+change every time to see if it should be built, whereas project watches allow us
+to treat these as semi-push notifications (we still have to poll gmail).
+
+One drawback to this approach is that it's a hassle to set up the project
+watches for a large number of projects. Since bionicbb is only interested in a
+small subset of projects, this is a non-issue.
+
+If the buildbot has applied Verified-1 to a patchset, the user may add their own
+Verified+1 to the change and the buildbot will remove its rejection the next
+time the services polls (by default, every five minutes).
+
+The service will also listen for the following commands:
+
+ * `bionicbb:clean`: Something is very broken and the buildbot's output
+ directory needs to be nuked.
+ * `bionicbb:retry`: Something went wrong and the buildbot should retry the
+ build.
+
+build\_listener.py
+------------------
+
+The build listener service responds to HTTP POST events sent from Jenkins and
+updates CLs accordingly. The only other API endpoint is `/drop-rejection`, which
+will remove a Verified-1 from a previously rejected patchset. The actually
+invocation of this is handled by the gmail listener.
diff --git a/tools/bionicbb/__init__.py b/tools/bionicbb/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tools/bionicbb/build_listener.py b/tools/bionicbb/build_listener.py
new file mode 100644
index 000000000..3a0032d5e
--- /dev/null
+++ b/tools/bionicbb/build_listener.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python2
+# pylint: disable=bad-indentation
+# vim: set sw=2 ts=2:
+import json
+import requests
+import termcolor
+
+import bionicbb.gerrit
+
+from flask import Flask, request
+app = Flask(__name__)
+
+
+def gerrit_url(endpoint):
+ gerrit_base_url = 'https://android-review.googlesource.com'
+ return gerrit_base_url + endpoint
+
+
+@app.route('/', methods=['POST'])
+def handle_build_message():
+ result = json.loads(request.data)
+
+ name = result['name']
+ number = result['build']['number']
+ status = result['build']['status']
+ go_url = 'http://go/bionicbb/' + result['build']['url']
+ full_url = result['build']['full_url']
+ params = result['build']['parameters']
+ change_id = params['CHANGE_ID']
+ ref = params['REF']
+ patch_set = ref.split('/')[-1]
+
+ print '{} #{} {}: {}'.format(name, number, status, full_url)
+
+ # bionic-lint is always broken, so we don't want to reject changes for those
+ # failures until we clean things up.
+ if name == 'bionic-presubmit':
+ message_lines = ['{} #{} checkbuild {}: {}'.format(
+ name, number, status, go_url)]
+ if status == 'FAILURE':
+ message_lines += ['If you believe this Verified-1 was in error, +1 the '
+ 'change and bionicbb will remove the -1 shortly.']
+
+ request_data = {
+ 'message': '\n'.join(message_lines)
+ }
+
+ label = 'Verified'
+ if status == 'FAILURE':
+ request_data['labels'] = {label: -1}
+ elif status == 'SUCCESS':
+ request_data['labels'] = {label: +1}
+
+ url = gerrit_url('/a/changes/{}/revisions/{}/review'.format(change_id,
+ patch_set))
+
+ headers = {'Content-Type': 'application/json;charset=UTF-8'}
+ print 'POST {}: {}'.format(url, request_data)
+ print requests.post(url, headers=headers, json=request_data)
+ elif name == 'clean-bionic-presubmit':
+ request_data = {'message': 'out/ directory removed'}
+ url = gerrit_url('/a/changes/{}/revisions/{}/review'.format(change_id,
+ patch_set))
+ headers = {'Content-Type': 'application/json;charset=UTF-8'}
+ print 'POST {}: {}'.format(url, request_data)
+ print requests.post(url, headers=headers, json=request_data)
+ elif name == 'bionic-lint':
+ print 'IGNORED'
+ else:
+ print '{}: {}'.format(termcolor.colored('red', 'UNKNOWN'), name)
+ return ''
+
+
+@app.route('/drop-rejection', methods=['POST'])
+def drop_rejection():
+ revision_info = json.loads(request.data)
+
+ change_id = revision_info['changeid']
+ patch_set = revision_info['patchset']
+
+ bb_email = 'bionicbb@android.com'
+ labels = bionicbb.gerrit.get_labels(change_id, patch_set)
+ if bb_email in labels['Verified']:
+ bb_review = labels['Verified'][bb_email]
+ else:
+ bb_review = 0
+
+ if bb_review >= 0:
+ print 'No rejection to drop: {} {}'.format(change_id, patch_set)
+ return ''
+
+ print 'Dropping rejection: {} {}'.format(change_id, patch_set)
+
+ request_data = {'labels': {'Verified': 0}}
+ url = gerrit_url('/a/changes/{}/revisions/{}/review'.format(change_id,
+ patch_set))
+ headers = {'Content-Type': 'application/json;charset=UTF-8'}
+ print 'POST {}: {}'.format(url, request_data)
+ print requests.post(url, headers=headers, json=request_data)
+ return ''
+
+
+if __name__ == "__main__":
+ app.run(host='0.0.0.0', debug=True)
diff --git a/tools/bionicbb/gerrit.py b/tools/bionicbb/gerrit.py
new file mode 100644
index 000000000..51df4fbce
--- /dev/null
+++ b/tools/bionicbb/gerrit.py
@@ -0,0 +1,56 @@
+# pylint: disable=bad-indentation
+# vim: set sw=2 ts=2:
+import json
+import requests
+
+
+class GerritError(RuntimeError):
+ def __init__(self, code, url):
+ self.code = code
+ self.url = url
+ super(GerritError, self).__init__('Error {}: {}'.format(code, url))
+
+
+def call(endpoint, method='GET'):
+ if method != 'GET':
+ raise NotImplementedError('Currently only HTTP GET is supported.')
+ gerrit_url = 'https://android-review.googlesource.com'
+ url = gerrit_url + endpoint
+ response = requests.get(url)
+ if response.status_code != 200:
+ raise GerritError(response.status_code, url)
+ return response.text[5:]
+
+
+def ref_for_change(change_id):
+ endpoint = '/changes/{}/detail?o=CURRENT_REVISION'.format(change_id)
+ change = json.loads(call(endpoint))
+ commit = change['current_revision']
+ return change['revisions'][commit]['fetch']['http']['ref']
+
+
+def get_labels(change_id, patch_set):
+ """Returns labels attached to a revision.
+
+ Returned data is in the following format:
+ {
+ 'Code-Review': {
+ <email>: <value>,
+ ...
+ },
+ 'Verified': {
+ <email>: <value>,
+ ...
+ }
+ }
+ """
+ details = call('/changes/{}/revisions/{}/review'.format(
+ change_id, patch_set))
+ labels = {'Code-Review': {}, 'Verified': {}}
+ for review in details['labels']['Code-Review']['all']:
+ if 'value' in review and 'email' in review:
+ labels['Code-Review'][review['email']] = int(review['value'])
+ for review in details['labels']['Verified']['all']:
+ if 'value' in review and 'email' in review:
+ labels['Verified'][review['email']] = int(review['value'])
+ return labels
diff --git a/tools/bionicbb/gmail_listener.py b/tools/bionicbb/gmail_listener.py
new file mode 100644
index 000000000..e7d0d4d62
--- /dev/null
+++ b/tools/bionicbb/gmail_listener.py
@@ -0,0 +1,314 @@
+#!/usr/bin/env python2
+# pylint: disable=bad-indentation
+# vim: set sw=2 ts=2:
+import base64
+import httplib
+import httplib2
+import jenkinsapi
+import json
+import re
+import requests
+import termcolor
+import socket
+import sys
+import time
+
+import apiclient.errors
+
+import bionicbb.config
+import bionicbb.gerrit
+
+
+class GmailError(RuntimeError):
+ def __init__(self, message):
+ super(GmailError, self).__init__(message)
+
+
+def get_gerrit_label(labels):
+ for label in labels:
+ if label['name'] == 'gerrit':
+ return label['id']
+ return None
+
+
+def get_headers(msg):
+ headers = {}
+ for hdr in msg['payload']['headers']:
+ headers[hdr['name']] = hdr['value']
+ return headers
+
+
+def build_service():
+ from apiclient.discovery import build
+ from oauth2client.client import flow_from_clientsecrets
+ from oauth2client.file import Storage
+ from oauth2client.tools import run
+
+ OAUTH_SCOPE = 'https://www.googleapis.com/auth/gmail.modify'
+ STORAGE = Storage('oauth.storage')
+
+ # Start the OAuth flow to retrieve credentials
+ flow = flow_from_clientsecrets(bionicbb.config.client_secret_file,
+ scope=OAUTH_SCOPE)
+ http = httplib2.Http()
+
+ # Try to retrieve credentials from storage or run the flow to generate them
+ credentials = STORAGE.get()
+ if credentials is None or credentials.invalid:
+ credentials = run(flow, STORAGE, http=http)
+
+ http = credentials.authorize(http)
+ return build('gmail', 'v1', http=http)
+
+
+def get_all_messages(service, label):
+ msgs = []
+ response = service.users().messages().list(
+ userId='me', labelIds=label).execute()
+ if 'messages' in response:
+ msgs.extend(response['messages'])
+ while 'nextPageToken' in response:
+ page_token = response['nextPageToken']
+ response = service.users().messages().list(
+ userId='me', pageToken=page_token).execute()
+ msgs.extend(response['messages'])
+ return msgs
+
+
+def get_body(msg):
+ if 'attachmentId' in msg['payload']['body']:
+ raise NotImplementedError('Handling of messages contained in '
+ 'attachments not yet implemented.')
+ b64_body = msg['payload']['body']['data']
+ return base64.urlsafe_b64decode(b64_body.encode('ASCII'))
+
+
+def get_gerrit_info(body):
+ info = {}
+ gerrit_pattern = r'^Gerrit-(\S+): (.+)$'
+ for match in re.finditer(gerrit_pattern, body, flags=re.MULTILINE):
+ info[match.group(1)] = match.group(2).strip()
+ return info
+
+
+def clean_project(gerrit_info, dry_run):
+ username = bionicbb.config.jenkins_credentials['username']
+ password = bionicbb.config.jenkins_credentials['password']
+ # TODO(danalbert): Move Jenkins server URL into config.py.
+ jenkins_url = 'http://bionicbb.mtv.corp.google.com:8080'
+ jenkins = jenkinsapi.api.Jenkins(jenkins_url, username, password)
+
+ build = 'clean-bionic-presubmit'
+ if build in jenkins:
+ if not dry_run:
+ job = jenkins[build].invoke()
+ url = job.get_build().baseurl
+ else:
+ url = 'DRY_RUN_URL'
+ print '{}({}): {} {}'.format(
+ termcolor.colored('CLEAN', 'green'),
+ gerrit_info['MessageType'],
+ build,
+ url)
+ else:
+ print '{}({}): {}'.format(
+ termcolor.colored('CLEAN', 'red'),
+ gerrit_info['MessageType'],
+ termcolor.colored(build, 'red'))
+ return True
+
+
+def build_project(gerrit_info, dry_run):
+ project_to_jenkins_map = {
+ 'platform/bionic': 'bionic-presubmit',
+ 'platform/build': 'bionic-presubmit',
+ 'platform/external/jemalloc': 'bionic-presubmit',
+ 'platform/external/libcxx': 'bionic-presubmit',
+ 'platform/external/libcxxabi': 'bionic-presubmit',
+ 'platform/external/compiler-rt': 'bionic-presubmit',
+ }
+
+ username = bionicbb.config.jenkins_credentials['username']
+ password = bionicbb.config.jenkins_credentials['password']
+ jenkins_url = 'http://bionicbb.mtv.corp.google.com:8080'
+ jenkins = jenkinsapi.api.Jenkins(jenkins_url, username, password)
+
+ project = gerrit_info['Project']
+ change_id = gerrit_info['Change-Id']
+ if project in project_to_jenkins_map:
+ build = project_to_jenkins_map[project]
+ else:
+ build = 'bionic-presubmit'
+
+ if build in jenkins:
+ project_path = '/'.join(project.split('/')[1:])
+ if not project_path:
+ raise RuntimeError('bogus project: {}'.format(project))
+ if project_path.startswith('platform/'):
+ print '{}({}): {} => {}'.format(
+ termcolor.colored('ERROR', 'red'),
+ 'project',
+ project,
+ project_path)
+ return False
+ try:
+ ref = bionicbb.gerrit.ref_for_change(change_id)
+ except bionicbb.gerrit.GerritError as ex:
+ print '{}({}): {} {}'.format(
+ termcolor.colored('GERRIT-ERROR', 'red'),
+ ex.code,
+ change_id,
+ ex.url)
+ return False
+ params = {
+ 'REF': ref,
+ 'CHANGE_ID': change_id,
+ 'PROJECT': project_path
+ }
+ if not dry_run:
+ job = jenkins[build].invoke(build_params=params)
+ url = job.get_build().baseurl
+ else:
+ url = 'DRY_RUN_URL'
+ print '{}({}): {} => {} {} {}'.format(
+ termcolor.colored('BUILD', 'green'),
+ gerrit_info['MessageType'],
+ project,
+ build,
+ url,
+ change_id)
+ else:
+ print '{}({}): {} => {} {}'.format(
+ termcolor.colored('BUILD', 'red'),
+ gerrit_info['MessageType'],
+ project,
+ termcolor.colored(build, 'red'),
+ change_id)
+ return True
+
+
+def handle_change(gerrit_info, _, dry_run):
+ return build_project(gerrit_info, dry_run)
+handle_newchange = handle_change
+handle_newpatchset = handle_change
+
+
+def drop_rejection(gerrit_info, dry_run):
+ request_data = {
+ 'changeid': gerrit_info['Change-Id'],
+ 'patchset': gerrit_info['PatchSet']
+ }
+ # TODO(danalbert): Move the URL for the build listener service into
+ # config.py.
+ url = 'http://bionicbb.mtv.corp.google.com:5000/drop-rejection'
+ headers = {'Content-Type': 'application/json;charset=UTF-8'}
+ if not dry_run:
+ try:
+ requests.post(url, headers=headers, data=json.dumps(request_data))
+ except requests.exceptions.ConnectionError as ex:
+ print '{}(drop-rejection): {}'.format(
+ termcolor.colored('ERROR', 'red'), ex)
+ return False
+ print '{}({}): {}'.format(
+ termcolor.colored('CHECK', 'green'),
+ gerrit_info['MessageType'],
+ gerrit_info['Change-Id'])
+ return True
+
+
+def handle_comment(gerrit_info, body, dry_run):
+ if 'Verified+1' in body:
+ drop_rejection(gerrit_info, dry_run)
+
+ command_map = {
+ 'clean': lambda: clean_project(gerrit_info, dry_run),
+ 'retry': lambda: build_project(gerrit_info, dry_run),
+ }
+
+ def handle_unknown_command():
+ pass # TODO(danalbert): should complain to the commenter.
+
+ commands = [match.group(1).strip() for match in
+ re.finditer(r'^bionicbb:\s*(.+)$', body, flags=re.MULTILINE)]
+
+ for command in commands:
+ if command in command_map:
+ command_map[command]()
+ else:
+ handle_unknown_command()
+
+ return True
+
+
+def skip_handler(gerrit_info, _, __):
+ print '{}({}): {}'.format(
+ termcolor.colored('SKIP', 'yellow'),
+ gerrit_info['MessageType'],
+ gerrit_info['Change-Id'])
+ return True
+handle_abandon = skip_handler
+handle_merged = skip_handler
+handle_restore = skip_handler
+handle_revert = skip_handler
+
+
+def process_message(msg, dry_run):
+ try:
+ body = get_body(msg)
+ gerrit_info = get_gerrit_info(body)
+ if not gerrit_info:
+ print termcolor.colored('No info found: {}'.format(msg['id']), 'red')
+ msg_type = gerrit_info['MessageType']
+ handler = 'handle_{}'.format(gerrit_info['MessageType'])
+ if handler in globals():
+ return globals()[handler](gerrit_info, body, dry_run)
+ else:
+ print termcolor.colored(
+ 'MessageType {} unhandled.'.format(msg_type), 'red')
+ print
+ return False
+ except NotImplementedError as ex:
+ print ex
+ return False
+
+
+def main(argc, argv):
+ dry_run = False
+ if argc == 2 and argv[1] == '--dry-run':
+ dry_run = True
+ elif argc > 2:
+ sys.exit('usage: python {} [--dry-run]'.format(argv[0]))
+
+ gmail_service = build_service()
+ msg_service = gmail_service.users().messages()
+
+ while True:
+ try:
+ labels = gmail_service.users().labels().list(userId='me').execute()
+ if not labels['labels']:
+ raise GmailError('Could not retrieve Gmail labels')
+ label_id = get_gerrit_label(labels['labels'])
+ if not label_id:
+ raise GmailError('Could not find gerrit label')
+
+ for msg in get_all_messages(gmail_service, label_id):
+ msg = msg_service.get(userId='me', id=msg['id']).execute()
+ if process_message(msg, dry_run) and not dry_run:
+ msg_service.trash(userId='me', id=msg['id']).execute()
+ time.sleep(60 * 5)
+ except GmailError as ex:
+ print '{}: {}!'.format(termcolor.colored('ERROR', 'red'), ex)
+ time.sleep(60 * 5)
+ except apiclient.errors.HttpError as ex:
+ print '{}: {}!'.format(termcolor.colored('ERROR', 'red'), ex)
+ time.sleep(60 * 5)
+ except httplib.BadStatusLine:
+ pass
+ except httplib2.ServerNotFoundError:
+ pass
+ except socket.error:
+ pass
+
+
+if __name__ == '__main__':
+ main(len(sys.argv), sys.argv)
diff --git a/tools/bionicbb/setup.cfg b/tools/bionicbb/setup.cfg
new file mode 100644
index 000000000..47cd585fa
--- /dev/null
+++ b/tools/bionicbb/setup.cfg
@@ -0,0 +1,2 @@
+[pep8]
+ignore = E111