Rewired the oath2 symlink and updated tgrid_parser to the new Build Bot version's tgrid syntax.

Added back the gviz_api to the tools deps - will remove it from the main project DEPS in a different patch. Rewired those symlinks too.

Made the build status loader algorithm more scalable. It read in the whole database on each load which is probably unsustainable in the long run. Also, it will now forget bots that have offline for more than ~5 revisions.

Fixed a bug in the coverage tracker.

BUG=
TEST=

Review URL: https://webrtc-codereview.appspot.com/486002

git-svn-id: http://webrtc.googlecode.com/svn/trunk@2039 4adac7df-926f-26a2-2b94-8c16560cd09d

tools/.gitignore

@@ -13,6 +13,7 @@
 /continuous_build/depot_tools
 /third_party/gaeunit
 /third_party/oauth2
+/third_party/google-visualization-python
 
 # Buildbot creates these files:
 .manhole

tools/DEPS

@@ -22,6 +22,10 @@ deps = {
   # Chromium buildbot scripts needs their own depot_tools.
   "tools/continuous_build/depot_tools":
     Var("chromium_trunk") + "/tools/depot_tools",
+
+  # Used by tools/quality_tracking/dashboard and tools/python_charts.
+  "tools/third_party/google-visualization-python":
+     "http://google-visualization-python.googlecode.com/svn/trunk/@15",
 }
 
 hooks = [

tools/quality_tracking/dashboard/add_coverage_data.py

@@ -77,7 +77,7 @@ class AddCoverageData(oauth_post_request_handler.OAuthPostRequestHandler):
 
   def _parse_percentage(self, key):
     """Parses out a percentage value from the request."""
-    value = float(self.request.get(key))
+    percentage = float(self.request.get(key))
     if percentage < 0.0 or percentage > 100.0:
       raise ValueError('%s is not a valid percentage.' % string_value)
     return percentage

tools/quality_tracking/dashboard/gviz_api.py

@@ -1 +1 @@
-../../../third_party/google-visualization-python/gviz_api.py
+../../third_party/google-visualization-python/gviz_api.py

tools/quality_tracking/dashboard/load_build_status.py

@@ -38,10 +38,18 @@ class BuildStatusLoader:
 
        The statuses OK, failed and warnings are considered to be conclusive.
 
-       The two most recent revisions are considered. The set of bots returned
-       will therefore be the bots that were reported the two most recent
-       revisions. This script will therefore adapt automatically to any changes
-       in the set of available bots.
+       The algorithm looks at the 100 most recent status entries, which should
+       give data on roughly the last five revisions if the number of bots stay
+       around 20 (The number 100 should be increased if the number of bots
+       increases significantly). This should give us enough data to get a
+       conclusive build status for all active bots.
+
+       With this limit, the algorithm will adapt automatically if a bot is
+       decommissioned - it will eventually disappear. The limit should not be
+       too high either since we will perhaps remember offline bots too long,
+       which could be confusing. The algorithm also adapts automatically to new
+       bots - these show up immediately if they get a build status for a recent
+       revision.
 
        Returns:
            A list of BuildStatusData entities with one entity per bot.
@@ -49,7 +57,8 @@ class BuildStatusLoader:
 
     build_status_entries = db.GqlQuery('SELECT * '
                                        'FROM BuildStatusData '
-                                       'ORDER BY revision DESC ')
+                                       'ORDER BY revision DESC '
+                                       'LIMIT 100')
 
     bots_to_latest_conclusive_entry = dict()
     for entry in build_status_entries:
@@ -85,7 +94,8 @@ class BuildStatusLoader:
 
         Implementation note: The data store fetches stuff as we go, so we won't
         read in the whole status table unless the LKGR is right at the end or
-        we don't have a LKGR.
+        we don't have a LKGR. Bots that are offline do not affect the LKGR
+        computation (e.g. they are not considered to be failed).
     """
     build_status_entries = db.GqlQuery('SELECT * '
                                        'FROM BuildStatusData '

tools/quality_tracking/oauth2

@@ -1 +1 @@
-../../third_party/oauth2/oauth2/
+../third_party/oauth2/oauth2

tools/quality_tracking/tgrid_parser.py

@@ -8,19 +8,27 @@
 #  in the file PATENTS.  All contributing project authors may
 #  be found in the AUTHORS file in the root of the source tree.
 
-"""Contains functions for parsing the build master's transposed grid page."""
+"""Contains functions for parsing the build master's transposed grid page.
+
+   Compatible with build bot 0.8.4 P1.
+"""
 
 __author__ = 'phoglund@webrtc.org (Patrik Höglund)'
 
 import re
 
 
+# This is here to work around a buggy build bot status message which makes no
+# sense, but which means the build failed when the slave was lost.
+BB_084_P1_BUGGY_STATUS = 'build<br/>successful<br/>exception<br/>slave<br/>lost'
+
+
 class FailedToParseBuildStatus(Exception):
   pass
 
 
 def _map_status(status):
-  if status == 'exception':
+  if status == 'exception' or status == BB_084_P1_BUGGY_STATUS:
     return 'failed'
   return status
 
@@ -38,8 +46,9 @@ def _parse_builds(revision, html):
   result = {}
 
   for match in re.finditer('<td.*?>.*?<a href="builders/(.+?)/builds/(\d+)">'
-                           '(OK|failed|building|warnings|exception)'
-                           '</a>.*?</td>',
+                           '(OK|failed|building|warnings|exception|' +
+                           BB_084_P1_BUGGY_STATUS + ')'
+                           '.*?</a>.*?</td>',
                            html, re.DOTALL):
     revision_and_bot_name = revision + "--" + match.group(1)
     build_number_and_status = match.group(2) + "--" + _map_status(
@@ -72,7 +81,7 @@ def parse_tgrid_page(html):
   """
   result = {}
 
-  for match in re.finditer('<td.*?class="sourcestamp">(\d+)</td>(.*?)</tr>',
+  for match in re.finditer('<td.*?class="sourcestamp">(\d+)  </td>(.*?)</tr>',
                            html, re.DOTALL):
     revision = match.group(1)
     builds_for_revision_html = match.group(2)

tools/quality_tracking/tgrid_parser_test.py

@@ -8,7 +8,10 @@
 #  in the file PATENTS.  All contributing project authors may
 #  be found in the AUTHORS file in the root of the source tree.
 
-"""Contains functions for parsing the build master's transposed grid page."""
+"""Test the tgrid parser.
+
+   Compatible with build bot 0.8.4 P1.
+"""
 
 __author__ = 'phoglund@webrtc.org (Patrik Höglund)'
 
@@ -18,89 +21,427 @@ import tgrid_parser
 
 
 SAMPLE_FILE = """
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 
-<html
- xmlns="http://www.w3.org/1999/xhtml"
- lang="en"
- xml:lang="en">
-<head>
- <title>Buildbot</title>
- <link href="buildbot.css" rel="stylesheet" type="text/css" />
-</head>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+            <title>Buildbot</title>
+    <link rel="stylesheet" href="default.css" type="text/css" />
+    <link rel="alternate" type="application/rss+xml" title="RSS" href="rss">
+      </head>
+  <body class="interface">
+    <div class="header">
+        <a href=".">Home</a>
+        - <a href="waterfall">Waterfall</a>
+        <a href="grid">Grid</a>
+        <a href="tgrid">T-Grid</a>
+        <a href="console">Console</a>
+        <a href="builders">Builders</a>
+        <a href="one_line_per_build">Recent Builds</a>
+        <a href="buildslaves">Buildslaves</a>
+        <a href="changes">Changesources</a>
+        - <a href="json/help">JSON API</a>
+        - <a href="about">About</a>
+    </div>
+    <hr/>
+
+    <div class="content">
+<h1>Transposed Grid View</h1>
 
-<body vlink="#800080">
 <table class="Grid" border="0" cellspacing="0">
+
 <tr>
-<td valign="bottom" class="sourcestamp">1570</td>
-<td class="build warnings"><a href="builders/Chrome/builds/109">warnings</a>
-<br />
-make chrome</td>
-<td class="build success">
-  <a href="builders/Android/builds/121">OK</a></td>
-<td class="build success">
-  <a href="builders/ChromeOS/builds/578">OK</a></td>
-<td class="build success">
-  <a href="builders/Linux32bitDBG/builds/564">OK</a></td>
-<td class="build success">
-  <a href="builders/Linux32bitRelease/builds/684">OK</a></td>
-<td class="build success">
-  <a href="builders/Linux64bitDBG/builds/680">OK</a></td>
-<td class="build success">
-  <a href="builders/Linux64bitDBG-GCC4.6/builds/5">OK</a></td>
-<td class="build success">
-  <a href="builders/Linux64bitRelease/builds/570">OK</a></td>
-<td class="build success">
-  <a href="builders/LinuxCLANG/builds/259">OK</a></td>
-<td class="build success">
-  <a href="builders/LinuxVideoTest/builds/345">OK</a></td>
-<td class="build success">
-  <a href="builders/MacOS/builds/670">OK</a></td>
-<td class="build success">
-  <a href="builders/Win32Debug/builds/432">OK</a></td>
-<td class="build success">
-  <a href="builders/Win32Release/builds/440">OK</a></td>
-</tr>
-<tr>
-<td valign="bottom" class="sourcestamp">1571</td>
-<td class="build warnings"><a href="builders/Chrome/builds/109">warnings</a>
-<br />
-make chrome</td>
-<td class="build success">
-  <a href="builders/Android/builds/122">OK</a></td>
-<td class="build success">
-  <a href="builders/ChromeOS/builds/579">OK</a></td>
-<td class="build success">
-  <a href="builders/Linux32bitDBG/builds/565">OK</a></td>
-<td class="build success">
-  <a href="builders/Linux32bitRelease/builds/685">OK</a></td>
-<td class="build success">
-  <a href="builders/Linux64bitDBG/builds/681">OK</a></td>
-<td class="build success">
-  <a href="builders/Linux64bitDBG-GCC4.6/builds/6">OK</a></td>
-<td class="build success">
-  <a href="builders/Linux64bitRelease/builds/571">OK</a></td>
-<td class="build success">
-  <a href="builders/LinuxCLANG/builds/260">OK</a></td>
-<td class="build failure">
-  <a href="builders/LinuxVideoTest/builds/346">failed</a><br />
-voe_auto_test</td>
-<td class="build success">
-  <a href="builders/MacOS/builds/671">OK</a></td>
-<td class="build running">
-  <a href="builders/Win32Debug/builds/441">building</a></td>
-<td class="build success">
-  <a href="builders/Win32Release/builds/441">OK</a></td>
-</tr>
+ <td class="title"><a href="http://www.chromium.org">WebRTC</a>
+
+
+ </td>
+   <td valign="middle" style="text-align: center" class="builder idle">
+    <a href="builders/Android">Android</a></td>
+   <td valign="middle" style="text-align: center" class="builder idle">
+    <a href="builders/AndroidNDK">AndroidNDK</a></td>
+   <td valign="middle" style="text-align: center" class="builder idle">
+    <a href="builders/Chrome">Chrome</a></td>
+   <td valign="middle" style="text-align: center" class="builder idle">
+    <a href="builders/ChromeOS">ChromeOS</a></td>
+   <td valign="middle" style="text-align: center" class="builder idle">
+    <a href="builders/Linux32DBG">Linux32DBG</a></td>
+   <td valign="middle" style="text-align: center" class="builder idle">
+    <a href="builders/Linux32Release">Linux32Release</a></td>
+   <td valign="middle" style="text-align: center" class="builder idle">
+    <a href="builders/Linux64DBG">Linux64DBG</a></td>
+   <td valign="middle" style="text-align: center" class="builder idle">
+    <a href="builders/Linux64DBG-GCC4.6">Linux64DBG-GCC4.6</a></td>
+   <td valign="middle" style="text-align: center" class="builder idle">
+    <a href="builders/Linux64Release">Linux64Release</a></td>
+   <td valign="middle" style="text-align: center" class="builder idle">
+    <a href="builders/LinuxClang">LinuxClang</a></td>
+   <td valign="middle" style="text-align: center" class="builder idle">
+    <a href="builders/LinuxValgrind">LinuxValgrind</a></td>
+   <td valign="middle" style="text-align: center" class="builder idle">
+    <a href="builders/LinuxVideoTest">LinuxVideoTest</a></td>
+   <td valign="middle" style="text-align: center" class="builder idle">
+    <a href="builders/MacOS32DBG">MacOS32DBG</a></td>
+   <td valign="middle" style="text-align: center" class="builder building">
+    <a href="builders/MacOS32Release">MacOS32Release</a><br/>(building)</td>
+   <td valign="middle" style="text-align: center" class="builder idle">
+    <a href="builders/Win32Debug">Win32Debug</a></td>
+   <td valign="middle" style="text-align: center" class="builder building">
+    <a href="builders/Win32Release">Win32Release</a><br/>(building)</td>
+ </tr>
+
+ <tr>
+ <td valign="bottom" class="sourcestamp">2006  </td>
+      <td class="build success">
+    <a href="builders/Android/builds/482">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/AndroidNDK/builds/70">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Chrome/builds/243">warnings</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/ChromeOS/builds/933">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux32DBG/builds/936">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux32Release/builds/1050">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux64DBG/builds/1038">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux64DBG-GCC4.6/builds/371">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux64Release/builds/936">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/LinuxClang/builds/610">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/LinuxValgrind/builds/317">OK</a>
+  </td>
+
+      <td class="build">&nbsp;</td>
+
+      <td class="build success">
+    <a href="builders/MacOS32DBG/builds/1052">OK</a>
+  </td>
+
+      <td class="build">&nbsp;</td>
+
+      <td class="build success">
+    <a href="builders/Win32Debug/builds/822">OK</a>
+  </td>
+
+      <td class="build">&nbsp;</td>
+
+  </tr>
+ <tr>
+ <td valign="bottom" class="sourcestamp">2007  </td>
+      <td class="build success">
+    <a href="builders/Android/builds/483">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/AndroidNDK/builds/71">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Chrome/builds/244">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/ChromeOS/builds/934">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux32DBG/builds/937">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux32Release/builds/1051">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux64DBG/builds/1039">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux64DBG-GCC4.6/builds/372">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux64Release/builds/937">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/LinuxClang/builds/611">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/LinuxValgrind/builds/318">OK</a>
+  </td>
+
+      <td class="build failure">
+    <a href="builders/LinuxVideoTest/builds/731">failed<br/>voe_auto_test</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/MacOS32DBG/builds/1053">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/MacOS32Release/builds/309">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Win32Debug/builds/823">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Win32Release/builds/809">OK</a>
+  </td>
+
+  </tr>
+ <tr>
+ <td valign="bottom" class="sourcestamp">2008  </td>
+      <td class="build success">
+    <a href="builders/Android/builds/484">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/AndroidNDK/builds/72">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Chrome/builds/245">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/ChromeOS/builds/935">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux32DBG/builds/938">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux32Release/builds/1052">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux64DBG/builds/1040">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux64DBG-GCC4.6/builds/373">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux64Release/builds/938">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/LinuxClang/builds/612">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/LinuxValgrind/builds/319">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/LinuxVideoTest/builds/732">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/MacOS32DBG/builds/1054">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/MacOS32Release/builds/310">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Win32Debug/builds/824">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Win32Release/builds/810">OK</a>
+  </td>
+
+  </tr>
+ <tr>
+ <td valign="bottom" class="sourcestamp">2010  </td>
+      <td class="build success">
+    <a href="builders/Android/builds/485">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/AndroidNDK/builds/73">OK</a>
+  </td>
+
+      <td class="build">&nbsp;</td>
+
+      <td class="build success">
+    <a href="builders/ChromeOS/builds/936">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux32DBG/builds/939">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux32Release/builds/1053">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux64DBG/builds/1041">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux64DBG-GCC4.6/builds/374">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux64Release/builds/939">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/LinuxClang/builds/613">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/LinuxValgrind/builds/320">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/LinuxVideoTest/builds/733">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/MacOS32DBG/builds/1055">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/MacOS32Release/builds/311">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Win32Debug/builds/825">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Win32Release/builds/811">OK</a>
+  </td>
+
+  </tr>
+ <tr>
+ <td valign="bottom" class="sourcestamp">2011  </td>
+      <td class="build success">
+    <a href="builders/Android/builds/486">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/AndroidNDK/builds/74">OK</a>
+  </td>
+
+      <td class="build">&nbsp;</td>
+
+      <td class="build success">
+    <a href="builders/ChromeOS/builds/937">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux32DBG/builds/940">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux32Release/builds/1054">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux64DBG/builds/1042">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux64DBG-GCC4.6/builds/375">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Linux64Release/builds/940">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/LinuxClang/builds/614">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/LinuxValgrind/builds/321">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/LinuxVideoTest/builds/734">OK</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/MacOS32DBG/builds/1056">OK</a>
+  </td>
+
+      <td class="build running">
+    <a href="builders/MacOS32Release/builds/313">building</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Win32Debug/builds/826">OK</a>
+  </td>
+
+      <td class="build running">
+    <a href="builders/Win32Release/builds/813">building</a>
+  </td>
+
+  </tr>
+  <tr>
+    <td valign="bottom" class="sourcestamp">latest  </td>
+      <td class="build running">
+    <a href="builders/MacOS32Release/builds/313">building</a>
+  </td>
+
+      <td class="build success">
+    <a href="builders/Win32Debug/builds/826">OK</a>
+  </td>
+  </tr>
 </table>
-</body>
+
+</div><div class="footer" style="clear:both">
+      <hr/>
+      <a href="http://buildbot.net/">BuildBot</a> (0.8.4p1)
+      working for the&nbsp;<a href="http://www.chromium.org">WebRTC
+        </a>&nbsp;project.<br/>
+      Page built: <b>Thu 12 Apr 2012 03:49:32</b> (CDT)
+    </div>
+    </body>
 </html>
 """
 
 MINIMAL_OK = """
 <tr>
-<td valign="bottom" class="sourcestamp">1570</td>
+<td valign="bottom" class="sourcestamp">1570  </td>
 <td class="build success">
 <a href="builders/Android/builds/121">OK</a></td>
 </tr>
@@ -108,16 +449,16 @@ MINIMAL_OK = """
 
 MINIMAL_FAIL = """
 <tr>
-<td valign="bottom" class="sourcestamp">1573</td>
+<td valign="bottom" class="sourcestamp">1573  </td>
 <td class="build failure">
-<a href="builders/LinuxVideoTest/builds/347">failed</a><br />
-voe_auto_test</td>
+  <a href="builders/LinuxVideoTest/builds/731">failed<br/>voe_auto_test</a>
+</td>
 </tr>
 """
 
 MINIMAL_BUILDING = """
 <tr>
-<td valign="bottom" class="sourcestamp">1576</td>
+<td valign="bottom" class="sourcestamp">1576  </td>
 <td class="build running">
 <a href="builders/Win32Debug/builds/434">building</a></td>
 voe_auto_test</td>
@@ -126,7 +467,7 @@ voe_auto_test</td>
 
 MINIMAL_WARNED = """
 <tr>
-<td valign="bottom" class="sourcestamp">1576</td>
+<td valign="bottom" class="sourcestamp">1576  </td>
 <td class="build warnings">
 <a href="builders/Chrome/builds/109">warnings</a><br />
 make chrome</td>
@@ -135,13 +476,22 @@ make chrome</td>
 
 MINIMAL_EXCEPTION = """
 <tr>
-<td valign="bottom" class="sourcestamp">1576</td>
+<td valign="bottom" class="sourcestamp">1576  </td>
 <td class="build exception">
 <a href="builders/Chrome/builds/109">exception</a><br />
 Sync</td>
 </tr>
 """
 
+MINIMAL_EXCEPTION_SLAVE_LOST = """
+<tr>
+<td valign="bottom" class="sourcestamp">1576  </td>
+<td class="build retry">
+  <a href="builders/LinuxValgrind/builds/324">build<br/>successful<br/>exception<br/>slave<br/>lost</a>
+</td>
+</tr>
+"""
+
 class TGridParserTest(unittest.TestCase):
   def test_parser_throws_exception_on_empty_html(self):
     self.assertRaises(tgrid_parser.FailedToParseBuildStatus,
@@ -163,7 +513,7 @@ class TGridParserTest(unittest.TestCase):
     first_mapping = result.items()[0]
 
     self.assertEqual('1573--LinuxVideoTest', first_mapping[0])
-    self.assertEqual('347--failed', first_mapping[1])
+    self.assertEqual('731--failed', first_mapping[1])
 
   def test_parser_finds_building_bot(self):
     result = tgrid_parser.parse_tgrid_page(MINIMAL_BUILDING)
@@ -192,34 +542,51 @@ class TGridParserTest(unittest.TestCase):
     self.assertEqual('1576--Chrome', first_mapping[0])
     self.assertEqual('109--failed', first_mapping[1])
 
+  def test_parser_finds_exception_slave_lost_and_maps_to_failed(self):
+    # This is to work around a bug in build bot 0.8.4p1 where it may say that
+    # the build was successful AND the slave was lost. In this case the build
+    # is not actually successful, so treat it as such.
+    result = tgrid_parser.parse_tgrid_page(MINIMAL_EXCEPTION_SLAVE_LOST)
 
-  def test_parser_finds_all_bots_and_revisions(self):
+    self.assertEqual(1, len(result), 'There is only one bot in the sample.')
+    first_mapping = result.items()[0]
+
+    self.assertEqual('1576--LinuxValgrind', first_mapping[0])
+    self.assertEqual('324--failed', first_mapping[1])
+
+  def test_parser_finds_all_bots_and_revisions_except_forced_builds(self):
     result = tgrid_parser.parse_tgrid_page(SAMPLE_FILE)
 
-    # 2 * 13 = 26 bots in sample
-    self.assertEqual(26, len(result))
+    # 5*16 = 80 bots in sample. There's also five empty results because some
+    # bots did not run for some revisions, so 80 - 5 = 75 results. There are
+    # two additional statuses under an explicit 'latest' revision, which should
+    # be ignored since that means the build was forced.
+    self.assertEqual(75, len(result))
 
     # Make some samples
-    self.assertTrue(result.has_key('1570--ChromeOS'))
-    self.assertEquals('578--OK', result['1570--ChromeOS'])
+    self.assertTrue(result.has_key('2006--ChromeOS'))
+    self.assertEquals('933--OK', result['2006--ChromeOS'])
 
-    self.assertTrue(result.has_key('1570--Chrome'))
-    self.assertEquals('109--warnings', result['1570--Chrome'])
+    self.assertTrue(result.has_key('2006--Chrome'))
+    self.assertEquals('243--warnings', result['2006--Chrome'])
 
-    self.assertTrue(result.has_key('1570--LinuxCLANG'))
-    self.assertEquals('259--OK', result['1570--LinuxCLANG'])
+    self.assertTrue(result.has_key('2006--LinuxClang'))
+    self.assertEquals('610--OK', result['2006--LinuxClang'])
 
-    self.assertTrue(result.has_key('1570--Win32Release'))
-    self.assertEquals('440--OK', result['1570--Win32Release'])
+    # This one happened to not get reported in revision 2006, but it should be
+    # there in the next revision:
+    self.assertFalse(result.has_key('2006--Win32Release'))
+    self.assertTrue(result.has_key('2007--Win32Release'))
+    self.assertEquals('809--OK', result['2007--Win32Release'])
 
-    self.assertTrue(result.has_key('1571--ChromeOS'))
-    self.assertEquals('579--OK', result['1571--ChromeOS'])
+    self.assertTrue(result.has_key('2007--ChromeOS'))
+    self.assertEquals('934--OK', result['2007--ChromeOS'])
 
-    self.assertTrue(result.has_key('1571--LinuxVideoTest'))
-    self.assertEquals('346--failed', result['1571--LinuxVideoTest'])
+    self.assertTrue(result.has_key('2007--LinuxVideoTest'))
+    self.assertEquals('731--failed', result['2007--LinuxVideoTest'])
 
-    self.assertTrue(result.has_key('1571--Win32Debug'))
-    self.assertEquals('441--building', result['1571--Win32Debug'])
+    self.assertTrue(result.has_key('2011--Win32Release'))
+    self.assertEquals('813--building', result['2011--Win32Release'])
 
 
 if __name__ == '__main__':