am 11b3fa54: Merge "Check in bionicbb code."

* commit '11b3fa5432951950b3fc9aadfe56139969300cad':
  Check in bionicbb code.

tools/bionicbb/.gitignore

@@ -0,0 +1,56 @@
+config.py
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/

tools/bionicbb/README.md

@@ -0,0 +1,83 @@
+bionicbb
+========
+
+The bionic buildbot contains two services: a gmail polling service, and a web
+service that interacts with gerrit.
+
+Dependencies
+------------
+
+ * Python 2.7
+ * [Flask](http://flask.pocoo.org/)
+ * [Google API Client Library](https://developers.google.com/api-client-library/python/start/installation)
+ * [jenkinsapi](https://pypi.python.org/pypi/jenkinsapi)
+ * [Requests](http://docs.python-requests.org/en/latest/)
+ * [termcolor](https://pypi.python.org/pypi/termcolor)
+
+Setup
+-----
+
+Create a `config.py` in the same directory as the sources. The structure of the
+configuration file is as follows:
+
+    client_secret_file = 'CLIENT_SECRET_FILE.json'
+    jenkins_credentials = {
+        'username': 'JENKINS_USERNAME',
+        'password': 'JENKINS_PASSWORD',
+    }
+
+The client secret file comes from the Gmail API page of the [Google Developers
+Console](https://console.developers.google.com/). The Jenkins credentials are
+for a Jenkins account that has the appropriate permissions to launch the jobs
+the buildbot will use.
+
+You will also need to add the HTTP password for the buildbot's Gerrit account to
+`~/.netrc`. The HTTP password can be obtained from the [Gerrit HTTP password
+settings](https://android-review.googlesource.com/#/settings/http-password).
+
+To launch the services:
+
+    $ python build_listener.py >build.log 2>&1 &
+    $ python gmail_listener.py >mail.log 2>&1 &
+
+The mail listener will direct your browser to an authentication page for the
+Gmail API.
+
+gmail\_listener.py
+------------------
+
+Bionicbb polls a gmail account to find changes that need to be built. The gmail
+account needs to have a gerrit account set up with project watches on anything
+it finds interesting. This is a rather ugly hack, but it seems to be the
+simplest option available.
+
+Gerrit does offer a streaming notification service that would be _far_ better,
+but it is only available over an SSH conection to gerrit, and the AOSP gerrit
+does not support this connection.
+
+Another option would be polling gerrit itself, but we'd have to process each
+change every time to see if it should be built, whereas project watches allow us
+to treat these as semi-push notifications (we still have to poll gmail).
+
+One drawback to this approach is that it's a hassle to set up the project
+watches for a large number of projects. Since bionicbb is only interested in a
+small subset of projects, this is a non-issue.
+
+If the buildbot has applied Verified-1 to a patchset, the user may add their own
+Verified+1 to the change and the buildbot will remove its rejection the next
+time the services polls (by default, every five minutes).
+
+The service will also listen for the following commands:
+
+ * `bionicbb:clean`: Something is very broken and the buildbot's output
+   directory needs to be nuked.
+ * `bionicbb:retry`: Something went wrong and the buildbot should retry the
+   build.
+
+build\_listener.py
+------------------
+
+The build listener service responds to HTTP POST events sent from Jenkins and
+updates CLs accordingly. The only other API endpoint is `/drop-rejection`, which
+will remove a Verified-1 from a previously rejected patchset. The actually
+invocation of this is handled by the gmail listener.

tools/bionicbb/build_listener.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python2
+# pylint: disable=bad-indentation
+# vim: set sw=2 ts=2:
+import json
+import requests
+import termcolor
+
+import bionicbb.gerrit
+
+from flask import Flask, request
+app = Flask(__name__)
+
+
+def gerrit_url(endpoint):
+  gerrit_base_url = 'https://android-review.googlesource.com'
+  return gerrit_base_url + endpoint
+
+
+@app.route('/', methods=['POST'])
+def handle_build_message():
+  result = json.loads(request.data)
+
+  name = result['name']
+  number = result['build']['number']
+  status = result['build']['status']
+  go_url = 'http://go/bionicbb/' + result['build']['url']
+  full_url = result['build']['full_url']
+  params = result['build']['parameters']
+  change_id = params['CHANGE_ID']
+  ref = params['REF']
+  patch_set = ref.split('/')[-1]
+
+  print '{} #{} {}: {}'.format(name, number, status, full_url)
+
+  # bionic-lint is always broken, so we don't want to reject changes for those
+  # failures until we clean things up.
+  if name == 'bionic-presubmit':
+    message_lines = ['{} #{} checkbuild {}: {}'.format(
+        name, number, status, go_url)]
+    if status == 'FAILURE':
+      message_lines += ['If you believe this Verified-1 was in error, +1 the '
+                        'change and bionicbb will remove the -1 shortly.']
+
+    request_data = {
+        'message': '\n'.join(message_lines)
+    }
+
+    label = 'Verified'
+    if status == 'FAILURE':
+      request_data['labels'] = {label: -1}
+    elif status == 'SUCCESS':
+      request_data['labels'] = {label: +1}
+
+    url = gerrit_url('/a/changes/{}/revisions/{}/review'.format(change_id,
+                                                                patch_set))
+
+    headers = {'Content-Type': 'application/json;charset=UTF-8'}
+    print 'POST {}: {}'.format(url, request_data)
+    print requests.post(url, headers=headers, json=request_data)
+  elif name == 'clean-bionic-presubmit':
+    request_data = {'message': 'out/ directory removed'}
+    url = gerrit_url('/a/changes/{}/revisions/{}/review'.format(change_id,
+                                                                patch_set))
+    headers = {'Content-Type': 'application/json;charset=UTF-8'}
+    print 'POST {}: {}'.format(url, request_data)
+    print requests.post(url, headers=headers, json=request_data)
+  elif name == 'bionic-lint':
+    print 'IGNORED'
+  else:
+    print '{}: {}'.format(termcolor.colored('red', 'UNKNOWN'), name)
+  return ''
+
+
+@app.route('/drop-rejection', methods=['POST'])
+def drop_rejection():
+  revision_info = json.loads(request.data)
+
+  change_id = revision_info['changeid']
+  patch_set = revision_info['patchset']
+
+  bb_email = 'bionicbb@android.com'
+  labels = bionicbb.gerrit.get_labels(change_id, patch_set)
+  if bb_email in labels['Verified']:
+    bb_review = labels['Verified'][bb_email]
+  else:
+    bb_review = 0
+
+  if bb_review >= 0:
+    print 'No rejection to drop: {} {}'.format(change_id, patch_set)
+    return ''
+
+  print 'Dropping rejection: {} {}'.format(change_id, patch_set)
+
+  request_data = {'labels': {'Verified': 0}}
+  url = gerrit_url('/a/changes/{}/revisions/{}/review'.format(change_id,
+                                                              patch_set))
+  headers = {'Content-Type': 'application/json;charset=UTF-8'}
+  print 'POST {}: {}'.format(url, request_data)
+  print requests.post(url, headers=headers, json=request_data)
+  return ''
+
+
+if __name__ == "__main__":
+  app.run(host='0.0.0.0', debug=True)

tools/bionicbb/gerrit.py

@@ -0,0 +1,56 @@
+# pylint: disable=bad-indentation
+# vim: set sw=2 ts=2:
+import json
+import requests
+
+
+class GerritError(RuntimeError):
+  def __init__(self, code, url):
+    self.code = code
+    self.url = url
+    super(GerritError, self).__init__('Error {}: {}'.format(code, url))
+
+
+def call(endpoint, method='GET'):
+  if method != 'GET':
+    raise NotImplementedError('Currently only HTTP GET is supported.')
+  gerrit_url = 'https://android-review.googlesource.com'
+  url = gerrit_url + endpoint
+  response = requests.get(url)
+  if response.status_code != 200:
+    raise GerritError(response.status_code, url)
+  return response.text[5:]
+
+
+def ref_for_change(change_id):
+  endpoint = '/changes/{}/detail?o=CURRENT_REVISION'.format(change_id)
+  change = json.loads(call(endpoint))
+  commit = change['current_revision']
+  return change['revisions'][commit]['fetch']['http']['ref']
+
+
+def get_labels(change_id, patch_set):
+  """Returns labels attached to a revision.
+
+  Returned data is in the following format:
+  {
+      'Code-Review': {
+          <email>: <value>,
+          ...
+      },
+      'Verified': {
+          <email>: <value>,
+          ...
+      }
+  }
+  """
+  details = call('/changes/{}/revisions/{}/review'.format(
+      change_id, patch_set))
+  labels = {'Code-Review': {}, 'Verified': {}}
+  for review in details['labels']['Code-Review']['all']:
+    if 'value' in review and 'email' in review:
+      labels['Code-Review'][review['email']] = int(review['value'])
+  for review in details['labels']['Verified']['all']:
+    if 'value' in review and 'email' in review:
+      labels['Verified'][review['email']] = int(review['value'])
+  return labels

tools/bionicbb/gmail_listener.py

@@ -0,0 +1,314 @@
+#!/usr/bin/env python2
+# pylint: disable=bad-indentation
+# vim: set sw=2 ts=2:
+import base64
+import httplib
+import httplib2
+import jenkinsapi
+import json
+import re
+import requests
+import termcolor
+import socket
+import sys
+import time
+
+import apiclient.errors
+
+import bionicbb.config
+import bionicbb.gerrit
+
+
+class GmailError(RuntimeError):
+  def __init__(self, message):
+    super(GmailError, self).__init__(message)
+
+
+def get_gerrit_label(labels):
+  for label in labels:
+    if label['name'] == 'gerrit':
+      return label['id']
+  return None
+
+
+def get_headers(msg):
+  headers = {}
+  for hdr in msg['payload']['headers']:
+    headers[hdr['name']] = hdr['value']
+  return headers
+
+
+def build_service():
+  from apiclient.discovery import build
+  from oauth2client.client import flow_from_clientsecrets
+  from oauth2client.file import Storage
+  from oauth2client.tools import run
+
+  OAUTH_SCOPE = 'https://www.googleapis.com/auth/gmail.modify'
+  STORAGE = Storage('oauth.storage')
+
+  # Start the OAuth flow to retrieve credentials
+  flow = flow_from_clientsecrets(bionicbb.config.client_secret_file,
+                                 scope=OAUTH_SCOPE)
+  http = httplib2.Http()
+
+  # Try to retrieve credentials from storage or run the flow to generate them
+  credentials = STORAGE.get()
+  if credentials is None or credentials.invalid:
+    credentials = run(flow, STORAGE, http=http)
+
+  http = credentials.authorize(http)
+  return build('gmail', 'v1', http=http)
+
+
+def get_all_messages(service, label):
+  msgs = []
+  response = service.users().messages().list(
+      userId='me', labelIds=label).execute()
+  if 'messages' in response:
+    msgs.extend(response['messages'])
+  while 'nextPageToken' in response:
+    page_token = response['nextPageToken']
+    response = service.users().messages().list(
+        userId='me', pageToken=page_token).execute()
+    msgs.extend(response['messages'])
+  return msgs
+
+
+def get_body(msg):
+  if 'attachmentId' in msg['payload']['body']:
+    raise NotImplementedError('Handling of messages contained in '
+                              'attachments not yet implemented.')
+  b64_body = msg['payload']['body']['data']
+  return base64.urlsafe_b64decode(b64_body.encode('ASCII'))
+
+
+def get_gerrit_info(body):
+  info = {}
+  gerrit_pattern = r'^Gerrit-(\S+): (.+)$'
+  for match in re.finditer(gerrit_pattern, body, flags=re.MULTILINE):
+    info[match.group(1)] = match.group(2).strip()
+  return info
+
+
+def clean_project(gerrit_info, dry_run):
+  username = bionicbb.config.jenkins_credentials['username']
+  password = bionicbb.config.jenkins_credentials['password']
+  # TODO(danalbert): Move Jenkins server URL into config.py.
+  jenkins_url = 'http://bionicbb.mtv.corp.google.com:8080'
+  jenkins = jenkinsapi.api.Jenkins(jenkins_url, username, password)
+
+  build = 'clean-bionic-presubmit'
+  if build in jenkins:
+    if not dry_run:
+      job = jenkins[build].invoke()
+      url = job.get_build().baseurl
+    else:
+      url = 'DRY_RUN_URL'
+    print '{}({}): {} {}'.format(
+        termcolor.colored('CLEAN', 'green'),
+        gerrit_info['MessageType'],
+        build,
+        url)
+  else:
+    print '{}({}): {}'.format(
+        termcolor.colored('CLEAN', 'red'),
+        gerrit_info['MessageType'],
+        termcolor.colored(build, 'red'))
+  return True
+
+
+def build_project(gerrit_info, dry_run):
+  project_to_jenkins_map = {
+      'platform/bionic': 'bionic-presubmit',
+      'platform/build': 'bionic-presubmit',
+      'platform/external/jemalloc': 'bionic-presubmit',
+      'platform/external/libcxx': 'bionic-presubmit',
+      'platform/external/libcxxabi': 'bionic-presubmit',
+      'platform/external/compiler-rt': 'bionic-presubmit',
+  }
+
+  username = bionicbb.config.jenkins_credentials['username']
+  password = bionicbb.config.jenkins_credentials['password']
+  jenkins_url = 'http://bionicbb.mtv.corp.google.com:8080'
+  jenkins = jenkinsapi.api.Jenkins(jenkins_url, username, password)
+
+  project = gerrit_info['Project']
+  change_id = gerrit_info['Change-Id']
+  if project in project_to_jenkins_map:
+    build = project_to_jenkins_map[project]
+  else:
+    build = 'bionic-presubmit'
+
+  if build in jenkins:
+    project_path = '/'.join(project.split('/')[1:])
+    if not project_path:
+      raise RuntimeError('bogus project: {}'.format(project))
+    if project_path.startswith('platform/'):
+      print '{}({}): {} => {}'.format(
+          termcolor.colored('ERROR', 'red'),
+          'project',
+          project,
+          project_path)
+      return False
+    try:
+      ref = bionicbb.gerrit.ref_for_change(change_id)
+    except bionicbb.gerrit.GerritError as ex:
+      print '{}({}): {} {}'.format(
+          termcolor.colored('GERRIT-ERROR', 'red'),
+          ex.code,
+          change_id,
+          ex.url)
+      return False
+    params = {
+        'REF': ref,
+        'CHANGE_ID': change_id,
+        'PROJECT': project_path
+    }
+    if not dry_run:
+      job = jenkins[build].invoke(build_params=params)
+      url = job.get_build().baseurl
+    else:
+      url = 'DRY_RUN_URL'
+    print '{}({}): {} => {} {} {}'.format(
+        termcolor.colored('BUILD', 'green'),
+        gerrit_info['MessageType'],
+        project,
+        build,
+        url,
+        change_id)
+  else:
+    print '{}({}): {} => {} {}'.format(
+        termcolor.colored('BUILD', 'red'),
+        gerrit_info['MessageType'],
+        project,
+        termcolor.colored(build, 'red'),
+        change_id)
+  return True
+
+
+def handle_change(gerrit_info, _, dry_run):
+  return build_project(gerrit_info, dry_run)
+handle_newchange = handle_change
+handle_newpatchset = handle_change
+
+
+def drop_rejection(gerrit_info, dry_run):
+  request_data = {
+      'changeid': gerrit_info['Change-Id'],
+      'patchset': gerrit_info['PatchSet']
+  }
+  # TODO(danalbert): Move the URL for the build listener service into
+  # config.py.
+  url = 'http://bionicbb.mtv.corp.google.com:5000/drop-rejection'
+  headers = {'Content-Type': 'application/json;charset=UTF-8'}
+  if not dry_run:
+    try:
+      requests.post(url, headers=headers, data=json.dumps(request_data))
+    except requests.exceptions.ConnectionError as ex:
+      print '{}(drop-rejection): {}'.format(
+          termcolor.colored('ERROR', 'red'), ex)
+      return False
+  print '{}({}): {}'.format(
+      termcolor.colored('CHECK', 'green'),
+      gerrit_info['MessageType'],
+      gerrit_info['Change-Id'])
+  return True
+
+
+def handle_comment(gerrit_info, body, dry_run):
+  if 'Verified+1' in body:
+    drop_rejection(gerrit_info, dry_run)
+
+  command_map = {
+      'clean': lambda: clean_project(gerrit_info, dry_run),
+      'retry': lambda: build_project(gerrit_info, dry_run),
+  }
+
+  def handle_unknown_command():
+    pass  # TODO(danalbert): should complain to the commenter.
+
+  commands = [match.group(1).strip() for match in
+              re.finditer(r'^bionicbb:\s*(.+)$', body, flags=re.MULTILINE)]
+
+  for command in commands:
+    if command in command_map:
+      command_map[command]()
+    else:
+      handle_unknown_command()
+
+  return True
+
+
+def skip_handler(gerrit_info, _, __):
+  print '{}({}): {}'.format(
+      termcolor.colored('SKIP', 'yellow'),
+      gerrit_info['MessageType'],
+      gerrit_info['Change-Id'])
+  return True
+handle_abandon = skip_handler
+handle_merged = skip_handler
+handle_restore = skip_handler
+handle_revert = skip_handler
+
+
+def process_message(msg, dry_run):
+  try:
+    body = get_body(msg)
+    gerrit_info = get_gerrit_info(body)
+    if not gerrit_info:
+      print termcolor.colored('No info found: {}'.format(msg['id']), 'red')
+    msg_type = gerrit_info['MessageType']
+    handler = 'handle_{}'.format(gerrit_info['MessageType'])
+    if handler in globals():
+      return globals()[handler](gerrit_info, body, dry_run)
+    else:
+      print termcolor.colored(
+          'MessageType {} unhandled.'.format(msg_type), 'red')
+    print
+    return False
+  except NotImplementedError as ex:
+    print ex
+    return False
+
+
+def main(argc, argv):
+  dry_run = False
+  if argc == 2 and argv[1] == '--dry-run':
+    dry_run = True
+  elif argc > 2:
+    sys.exit('usage: python {} [--dry-run]'.format(argv[0]))
+
+  gmail_service = build_service()
+  msg_service = gmail_service.users().messages()
+
+  while True:
+    try:
+      labels = gmail_service.users().labels().list(userId='me').execute()
+      if not labels['labels']:
+        raise GmailError('Could not retrieve Gmail labels')
+      label_id = get_gerrit_label(labels['labels'])
+      if not label_id:
+        raise GmailError('Could not find gerrit label')
+
+      for msg in get_all_messages(gmail_service, label_id):
+        msg = msg_service.get(userId='me', id=msg['id']).execute()
+        if process_message(msg, dry_run) and not dry_run:
+          msg_service.trash(userId='me', id=msg['id']).execute()
+      time.sleep(60 * 5)
+    except GmailError as ex:
+      print '{}: {}!'.format(termcolor.colored('ERROR', 'red'), ex)
+      time.sleep(60 * 5)
+    except apiclient.errors.HttpError as ex:
+      print '{}: {}!'.format(termcolor.colored('ERROR', 'red'), ex)
+      time.sleep(60 * 5)
+    except httplib.BadStatusLine:
+      pass
+    except httplib2.ServerNotFoundError:
+      pass
+    except socket.error:
+      pass
+
+
+if __name__ == '__main__':
+  main(len(sys.argv), sys.argv)

tools/bionicbb/setup.cfg

@@ -0,0 +1,2 @@
+[pep8]
+ignore = E111