Implemented build status tracking.

Left to do:
- UI for presenting the data
- Directory structure reorganization

BUG=
TEST=

Review URL: https://webrtc-codereview.appspot.com/382003

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1588 4adac7df-926f-26a2-2b94-8c16560cd09d

tools/quality_tracking/constants.py

@@ -28,7 +28,7 @@ ACCESS_TOKEN_URL = DASHBOARD_SERVER_HTTP + '/_ah/OAuthGetAccessToken'
 
 # The build master URL.
 BUILD_MASTER_SERVER = 'webrtc-cb-linux-master.cbf.corp.google.com:8010'
-BUILD_MASTER_LATEST_BUILD_URL = '/one_box_per_builder'
+BUILD_MASTER_TRANSPOSED_GRID_URL = '/tgrid'
 
 # The build-bot user which runs build bot jobs.
 BUILD_BOT_USER = 'phoglund'

tools/quality_tracking/dashboard/add_build_status_data.py

@@ -16,15 +16,44 @@ from google.appengine.ext import db
 
 import oauth_post_request_handler
 
+VALID_STATUSES = ['OK', 'failed', 'building']
 
-SUCCESSFUL_STRING_TO_BOOLEAN = {'successful': True, 'failed': False}
+
+class OrphanedBuildStatusesExistException(Exception):
+  pass
+
+
+class BuildStatusRoot(db.Model):
+  """Exists solely to be the root parent for all build status data.
+
+     Since all build status data will refer to this as their parent,
+     we can run transactions on the build status data as a whole.
+  """
+  pass
 
 
 class BuildStatusData(db.Model):
   """This represents one build status report from the build bot."""
   bot_name = db.StringProperty(required=True)
+  revision = db.IntegerProperty(required=True)
   build_number = db.IntegerProperty(required=True)
-  successful = db.BooleanProperty(required=True)
+  status = db.StringProperty(required=True)
+
+
+def _ensure_build_status_root_exists():
+  root = db.GqlQuery('SELECT * FROM BuildStatusRoot').get()
+  if not root:
+    # Create a new root, but ensure we don't have any orphaned build statuses
+    # (in that case, we would not have a single entity group as we desire).
+    orphans = db.GqlQuery('SELECT * FROM BuildStatusData').get()
+    if orphans:
+      raise OrphanedBuildStatusesExistException('Parent is gone and there are '
+                                                'orphaned build statuses in '
+                                                'the database!')
+    root = BuildStatusRoot()
+    root.put()
+
+  return root
 
 
 def _filter_oauth_parameters(post_keys):
@@ -32,26 +61,96 @@ def _filter_oauth_parameters(post_keys):
                 post_keys)
 
 
+def _parse_status(build_number_and_status):
+  parsed_status = build_number_and_status.split('--')
+  if len(parsed_status) != 2:
+    raise ValueError('Malformed status string %s.' % build_number_and_status)
+
+  parsed_build_number = int(parsed_status[0])
+  status = parsed_status[1]
+
+  if status not in VALID_STATUSES:
+    raise ValueError('Invalid status in %s.' % build_number_and_status)
+
+  return (parsed_build_number, status)
+
+
+def _parse_name(revision_and_bot_name):
+  parsed_name = revision_and_bot_name.split('--')
+  if len(parsed_name) != 2:
+    raise ValueError('Malformed name string %s.' % revision_and_bot_name)
+
+  revision = parsed_name[0]
+  bot_name = parsed_name[1]
+
+  return (int(revision), bot_name)
+
+
+def _delete_all_with_revision(revision, build_status_root):
+  query_result = db.GqlQuery('SELECT * FROM BuildStatusData '
+                             'WHERE revision = :1 AND ANCESTOR IS :2',
+                             revision, build_status_root)
+  for entry in query_result:
+    entry.delete()
+
+
 class AddBuildStatusData(oauth_post_request_handler.OAuthPostRequestHandler):
-  """Used to report build status data."""
+  """Used to report build status data.
+
+     Build status data is reported as a POST request. The POST request, aside
+     from the required oauth_* parameters should contain name-value entries that
+     abide by the following rules:
+
+     1) The name should be on the form <revision>--<bot name>, for instance
+        1568--Win32Release.
+     2) The value should be on the form <build number>--<status>, for instance
+        553--OK, 554--building. The status is permitted to be failed, OK or
+        building.
+
+    Data is keyed by revision. This handler will delete all data from a revision
+    if data with that revision is present in the current update, since we
+    assume that more recent data is always better data. We also assume that
+    an update always has complete information on a revision (e.g. the status
+    for all the bots are reported in each update).
+
+    In particular the revision arrangement solves the problem when the latest
+    revision reports 'building' for a bot. Had we not deleted the old revision
+    we would first store a 'building' status for that bot and revision, and
+    later store a 'OK' or 'failed' status for that bot and revision. This is
+    undesirable since we don't want multiple statuses for one bot-revision
+    combination. Now we will effectively update the bot's status instead.
+  """
 
   def post(self):
-    for bot_name in _filter_oauth_parameters(self.request.arguments()):
+    build_status_root = _ensure_build_status_root_exists()
-      status = self.request.get(bot_name)
+    build_status_data = _filter_oauth_parameters(self.request.arguments())
-      parsed_status = status.split('-')
-      if len(parsed_status) != 2:
-        raise ValueError('Malformed status string %s for bot %s.' %
-                         (status, bot_name))
 
-      parsed_build_number = int(parsed_status[0])
+    db.run_in_transaction(self._parse_and_store_data_in_transaction,
-      successful = parsed_status[1]
+                          build_status_root, build_status_data)
 
-      if successful not in SUCCESSFUL_STRING_TO_BOOLEAN:
+  def _parse_and_store_data_in_transaction(self, build_status_root,
-        raise ValueError('Malformed status string %s for bot %s.' % (status,
+                                           build_status_data):
-                                                                     bot_name))
+    encountered_revisions = set()
-      parsed_successful = SUCCESSFUL_STRING_TO_BOOLEAN[successful]
+    for revision_and_bot_name in build_status_data:
+      build_number_and_status = self.request.get(revision_and_bot_name)
 
-      item = BuildStatusData(bot_name=bot_name,
+      try:
-                             build_number=parsed_build_number,
+        (build_number, status) = _parse_status(build_number_and_status)
-                             successful=parsed_successful)
+        (revision, bot_name) = _parse_name(revision_and_bot_name)
+      except ValueError as error:
+        self._show_error_page('Invalid parameter in request: %s.' % error)
+
+      if revision not in encountered_revisions:
+        # There's new data on this revision in this update, so clear all status
+        # entries with that revision. Only do this once when we first encounter
+        # the revision.
+        _delete_all_with_revision(revision, build_status_root)
+        encountered_revisions.add(revision)
+
+      # Finally, write the item.
+      item = BuildStatusData(parent=build_status_root,
+                             bot_name=bot_name,
+                             revision=revision,
+                             build_number=build_number,
+                             status=status)
       item.put()

tools/quality_tracking/tgrid_parser.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+#  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+#
+#  Use of this source code is governed by a BSD-style license
+#  that can be found in the LICENSE file in the root of the source
+#  tree. An additional intellectual property rights grant can be found
+#  in the file PATENTS.  All contributing project authors may
+#  be found in the AUTHORS file in the root of the source tree.
+
+"""Contains functions for parsing the build master's transposed grid page."""
+
+__author__ = 'phoglund@webrtc.org (Patrik Höglund)'
+
+import re
+
+
+class FailedToParseBuildStatus(Exception):
+  pass
+
+
+def _parse_builds(revision, html):
+  """Parses the bot list, which is a sequence of <td></td> lines.
+
+     Example input:
+     <td class="build success"><a href="builders/Android/builds/119">OK</a></td>
+     The first regular expression group captures Android, second 119, third OK.
+  """
+  result = {}
+
+  for match in re.finditer('<td.*?>.*?<a href="builders/(.+?)/builds/(\d+)">'
+                           '(OK|failed|building)</a>.*?</td>', html, re.DOTALL):
+    revision_and_bot_name = revision + "--" + match.group(1)
+    build_number_and_status = match.group(2) + "--" + match.group(3)
+
+    result[revision_and_bot_name] = build_number_and_status
+
+  return result
+
+
+def parse_tgrid_page(html):
+  """Parses the build master's tgrid page.
+
+     Example input:
+     <tr>
+       <td valign="bottom" class="sourcestamp">1568</td>
+       LIST OF BOTS
+     </tr>
+     The first regular expression group captures 1568, second group captures
+     everything in LIST OF BOTS. The list of bots is then passed into a
+     separate function for parsing.
+
+     Args:
+         html: The raw HTML from the tgrid page.
+
+     Returns: A dictionary with <svn revision>--<bot name> mapped to
+         <bot build number>--<status>, where status is either OK, failed or
+         building.
+  """
+  result = {}
+
+  for match in re.finditer('<td.*?class="sourcestamp">(\d+)</td>(.*?)</tr>',
+                           html, re.DOTALL):
+    revision = match.group(1)
+    builds_for_revision_html = match.group(2)
+    result.update(_parse_builds(revision, builds_for_revision_html))
+
+  if not result:
+   raise FailedToParseBuildStatus('Could not find any build statuses in %s.' %
+                                  html)
+
+  return result

tools/quality_tracking/tgrid_parser_test.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+#  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+#
+#  Use of this source code is governed by a BSD-style license
+#  that can be found in the LICENSE file in the root of the source
+#  tree. An additional intellectual property rights grant can be found
+#  in the file PATENTS.  All contributing project authors may
+#  be found in the AUTHORS file in the root of the source tree.
+
+"""Contains functions for parsing the build master's transposed grid page."""
+
+__author__ = 'phoglund@webrtc.org (Patrik Höglund)'
+
+import unittest
+
+import tgrid_parser
+
+
+SAMPLE_FILE = """
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+
+<html
+ xmlns="http://www.w3.org/1999/xhtml"
+ lang="en"
+ xml:lang="en">
+<head>
+ <title>Buildbot</title>
+ <link href="buildbot.css" rel="stylesheet" type="text/css" />
+</head>
+
+<body vlink="#800080">
+<table class="Grid" border="0" cellspacing="0">
+<tr>
+<td valign="bottom" class="sourcestamp">1570</td>
+<td class="build success">
+  <a href="builders/Android/builds/121">OK</a></td>
+<td class="build success">
+  <a href="builders/ChromeOS/builds/578">OK</a></td>
+<td class="build success">
+  <a href="builders/Linux32bitDBG/builds/564">OK</a></td>
+<td class="build success">
+  <a href="builders/Linux32bitRelease/builds/684">OK</a></td>
+<td class="build success">
+  <a href="builders/Linux64bitDBG/builds/680">OK</a></td>
+<td class="build success">
+  <a href="builders/Linux64bitDBG-GCC4.6/builds/5">OK</a></td>
+<td class="build success">
+  <a href="builders/Linux64bitRelease/builds/570">OK</a></td>
+<td class="build success">
+  <a href="builders/LinuxCLANG/builds/259">OK</a></td>
+<td class="build success">
+  <a href="builders/LinuxVideoTest/builds/345">OK</a></td>
+<td class="build success">
+  <a href="builders/MacOS/builds/670">OK</a></td>
+<td class="build success">
+  <a href="builders/Win32Debug/builds/432">OK</a></td>
+<td class="build success">
+  <a href="builders/Win32Release/builds/440">OK</a></td>
+</tr>
+<tr>
+<td valign="bottom" class="sourcestamp">1571</td>
+<td class="build success">
+  <a href="builders/Android/builds/122">OK</a></td>
+<td class="build success">
+  <a href="builders/ChromeOS/builds/579">OK</a></td>
+<td class="build success">
+  <a href="builders/Linux32bitDBG/builds/565">OK</a></td>
+<td class="build success">
+  <a href="builders/Linux32bitRelease/builds/685">OK</a></td>
+<td class="build success">
+  <a href="builders/Linux64bitDBG/builds/681">OK</a></td>
+<td class="build success">
+  <a href="builders/Linux64bitDBG-GCC4.6/builds/6">OK</a></td>
+<td class="build success">
+  <a href="builders/Linux64bitRelease/builds/571">OK</a></td>
+<td class="build success">
+  <a href="builders/LinuxCLANG/builds/260">OK</a></td>
+<td class="build failure">
+  <a href="builders/LinuxVideoTest/builds/346">failed</a><br />
+voe_auto_test</td>
+<td class="build success">
+  <a href="builders/MacOS/builds/671">OK</a></td>
+<td class="build running">
+  <a href="builders/Win32Debug/builds/441">building</a></td>
+<td class="build success">
+  <a href="builders/Win32Release/builds/441">OK</a></td>
+</tr>
+</table>
+</body>
+</html>
+"""
+
+MINIMAL_OK = """
+<tr>
+<td valign="bottom" class="sourcestamp">1570</td>
+<td class="build success">
+<a href="builders/Android/builds/121">OK</a></td>
+</tr>
+"""
+
+MINIMAL_FAIL = """
+<tr>
+<td valign="bottom" class="sourcestamp">1573</td>
+<td class="build failure">
+<a href="builders/LinuxVideoTest/builds/347">failed</a><br />
+voe_auto_test</td>
+</tr>
+"""
+
+MINIMAL_BUILDING = """
+<tr>
+<td valign="bottom" class="sourcestamp">1576</td>
+<td class="build running">
+<a href="builders/Win32Debug/builds/434">building</a></td>
+voe_auto_test</td>
+</tr>
+"""
+
+class TGridParserTest(unittest.TestCase):
+  def test_parser_throws_exception_on_empty_html(self):
+    self.assertRaises(tgrid_parser.FailedToParseBuildStatus,
+                      tgrid_parser.parse_tgrid_page, '');
+
+  def test_parser_finds_successful_bot(self):
+    result = tgrid_parser.parse_tgrid_page(MINIMAL_OK)
+
+    self.assertEqual(1, len(result), 'There is only one bot in the sample.')
+    first_mapping = result.items()[0]
+
+    self.assertEqual('1570--Android', first_mapping[0])
+    self.assertEqual('121--OK', first_mapping[1])
+
+  def test_parser_finds_failed_bot(self):
+    result = tgrid_parser.parse_tgrid_page(MINIMAL_FAIL)
+
+    self.assertEqual(1, len(result), 'There is only one bot in the sample.')
+    first_mapping = result.items()[0]
+
+    self.assertEqual('1573--LinuxVideoTest', first_mapping[0])
+    self.assertEqual('347--failed', first_mapping[1])
+
+  def test_parser_finds_building_bot(self):
+    result = tgrid_parser.parse_tgrid_page(MINIMAL_BUILDING)
+
+    self.assertEqual(1, len(result), 'There is only one bot in the sample.')
+    first_mapping = result.items()[0]
+
+    self.assertEqual('1576--Win32Debug', first_mapping[0])
+    self.assertEqual('434--building', first_mapping[1])
+
+  def test_parser_finds_all_bots_and_revisions(self):
+    result = tgrid_parser.parse_tgrid_page(SAMPLE_FILE)
+
+    # 2 * 12 = 24 bots in sample
+    self.assertEqual(24, len(result))
+
+    # Make some samples
+    self.assertTrue(result.has_key('1570--ChromeOS'))
+    self.assertEquals('578--OK', result['1570--ChromeOS'])
+
+    self.assertTrue(result.has_key('1570--LinuxCLANG'))
+    self.assertEquals('259--OK', result['1570--LinuxCLANG'])
+
+    self.assertTrue(result.has_key('1570--Win32Release'))
+    self.assertEquals('440--OK', result['1570--Win32Release'])
+
+    self.assertTrue(result.has_key('1571--ChromeOS'))
+    self.assertEquals('579--OK', result['1571--ChromeOS'])
+
+    self.assertTrue(result.has_key('1571--LinuxVideoTest'))
+    self.assertEquals('346--failed', result['1571--LinuxVideoTest'])
+
+    self.assertTrue(result.has_key('1571--Win32Debug'))
+    self.assertEquals('441--building', result['1571--Win32Debug'])
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/quality_tracking/track_build_status.py

@@ -20,47 +20,16 @@ import re
 
 import constants
 import dashboard_connection
+import tgrid_parser
 
 
 class FailedToGetStatusFromMaster(Exception):
   pass
 
 
-class FailedToParseBuildStatus(Exception):
-  pass
-
-
-def _parse_status_page(html):
-  """Parses the build master's one_box_per_builder page.
-
-     Args:
-         html: The raw HTML from the one_box_per_builder page.
-
-     Returns: the bot name mapped to a string with the build number and the
-         build status separated by a dash (e.g. 456-successful, 114-failed).
-  """
-  result = {}
-
-  # Example target string: <a href="builders/Win32Debug/builds/430">#430</a>
-  #                        <br />build<br />successful</td>
-  # Group 1 captures 'Win32Debug', Group 2 captures '430', group 3 'successful'.
-  # Implementation note: We match non-greedily (.*?) between the link and
-  # successful / failed, otherwise we would only find the first status.
-  for match in re.finditer('<a href="builders/([^/]*)/builds/([0-9]+)">'
-                           '.*?(successful|failed)',
-                           html, re.DOTALL):
-    result[match.group(1)] = match.group(2) + '-' + match.group(3)
-
-  if not result:
-    raise FailedToParseBuildStatus('Could not find any build statuses in %s.' %
-                                   html)
-
-  return result
-
-
 def _download_and_parse_build_status():
   connection = httplib.HTTPConnection(constants.BUILD_MASTER_SERVER)
-  connection.request('GET', constants.BUILD_MASTER_LATEST_BUILD_URL)
+  connection.request('GET', constants.BUILD_MASTER_TRANSPOSED_GRID_URL)
   response = connection.getresponse()
 
   if response.status != 200:
@@ -71,7 +40,7 @@ def _download_and_parse_build_status():
   full_response = response.read()
   connection.close()
 
-  return _parse_status_page(full_response)
+  return tgrid_parser.parse_tgrid_page(full_response)
 
 
 def _main():