Cleaned up and completed current dashboard milestone.

Left to do:
- Directory reorganization.

BUG=
TEST=

Review URL: https://webrtc-codereview.appspot.com/384003

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1605 4adac7df-926f-26a2-2b94-8c16560cd09d

.gitignore

@@ -39,6 +39,7 @@
 /third_party/asan
 /third_party/cygwin
 /third_party/expat
+/third_party/gaeunit
 /third_party/google-gflags/src
 /third_party/google-visualization-python
 /third_party/jsoncpp

DEPS

@@ -84,13 +84,17 @@ deps = {
   "trunk/third_party/libyuv":
     (Var("googlecode_url") % "libyuv") + "/trunk@121",
 
-  # Used by tools/coverage/dashboard and tools/python_charts
+  # Used by tools/quality_tracking/dashboard and tools/python_charts
   "trunk/third_party/google-visualization-python":
     (Var("googlecode_url") % "google-visualization-python") + "/trunk@15",
 
-  # Used by tools/coverage
+  # Used by tools/quality_tracking
   "trunk/third_party/oauth2":
-    "https://github.com/simplegeo/python-oauth2.git@a83f4a297336b631e75cba102910c19231518159"
+    "https://github.com/simplegeo/python-oauth2.git@a83f4a29",
+
+  # Used by tools/quality_tracking
+  "trunk/third_party/gaeunit":
+    "https://code.google.com/p/gaeunit.git@e16d5bd4",
 }
 
 deps_os = {

tools/quality_tracking/dashboard/add_build_status_data.py

@@ -12,6 +12,8 @@
 
 __author__ = 'phoglund@webrtc.org (Patrik Höglund)'
 
+import datetime
+
 from google.appengine.ext import db
 
 import oauth_post_request_handler
@@ -24,12 +26,13 @@ class OrphanedBuildStatusesExistException(Exception):
 
 
 class BuildStatusRoot(db.Model):
-  """Exists solely to be the root parent for all build status data.
+  """Exists solely to be the root parent for all build status data and to keep
+     track of when the last update was made.
 
      Since all build status data will refer to this as their parent,
      we can run transactions on the build status data as a whole.
   """
-  pass
+  last_updated_at = db.DateTimeProperty()
 
 
 class BuildStatusData(db.Model):
@@ -121,7 +124,7 @@ class AddBuildStatusData(oauth_post_request_handler.OAuthPostRequestHandler):
     combination. Now we will effectively update the bot's status instead.
   """
 
-  def post(self):
+  def _parse_and_store_data(self):
     build_status_root = _ensure_build_status_root_exists()
     build_status_data = _filter_oauth_parameters(self.request.arguments())
 
@@ -130,6 +133,7 @@ class AddBuildStatusData(oauth_post_request_handler.OAuthPostRequestHandler):
 
   def _parse_and_store_data_in_transaction(self, build_status_root,
                                            build_status_data):
+
     encountered_revisions = set()
     for revision_and_bot_name in build_status_data:
       build_number_and_status = self.request.get(revision_and_bot_name)
@@ -154,3 +158,8 @@ class AddBuildStatusData(oauth_post_request_handler.OAuthPostRequestHandler):
                              build_number=build_number,
                              status=status)
       item.put()
+
+    request_posix_timestamp = float(self.request.get('oauth_timestamp'))
+    request_datetime = datetime.datetime.fromtimestamp(request_posix_timestamp)
+    build_status_root.last_updated_at = request_datetime
+    build_status_root.put()

tools/quality_tracking/dashboard/add_coverage_data.py

@@ -43,7 +43,7 @@ class AddCoverageData(oauth_post_request_handler.OAuthPostRequestHandler):
      function_coverage: A float percentage in the interval 0-100.0.
   """
 
-  def post(self):
+  def _parse_and_store_data(self):
     try:
       posix_time = int(self.request.get('date'))
       parsed_date = datetime.datetime.fromtimestamp(posix_time)

tools/quality_tracking/dashboard/app.yaml

@@ -2,8 +2,13 @@ application: dashboard
 version: 1
 runtime: python27
 api_version: 1
-threadsafe: true
+threadsafe: false
 
 handlers:
+- url: /stylesheets
+  static_dir: stylesheets
+# Note: tests should be disabled in production.
+# - url: /test.*
+#  script: gaeunit.py
 - url: /.*
   script: dashboard.app

tools/quality_tracking/dashboard/dashboard.py

@@ -12,12 +12,13 @@
 
 __author__ = 'phoglund@webrtc.org (Patrik Höglund)'
 
-from google.appengine.ext import db
-import gviz_api
+from google.appengine.ext.webapp import template
 import webapp2
 
 import add_build_status_data
 import add_coverage_data
+import load_build_status
+import load_coverage
 
 
 class ShowDashboard(webapp2.RequestHandler):
@@ -28,38 +29,18 @@ class ShowDashboard(webapp2.RequestHandler):
   """
 
   def get(self):
-    page_template_filename = 'templates/dashboard_template.html'
+    build_status_loader = load_build_status.BuildStatusLoader()
+    build_status_data = build_status_loader.load_build_status_data()
+    last_updated_at = build_status_loader.load_last_modified_at()
+    last_updated_at = last_updated_at.strftime("%Y-%m-%d %H:%M")
+    lkgr = build_status_loader.compute_lkgr()
 
-    # Load the page HTML template.
-    try:
-      template_file = open(page_template_filename)
-      page_template = template_file.read()
-      template_file.close()
-    except IOError as exception:
-      self._show_error_page('Cannot open page template file: %s<br>Details: %s'
-                            % (page_template_filename, exception))
-      return
-
-    coverage_entries = db.GqlQuery('SELECT * '
-                                   'FROM CoverageData '
-                                   'ORDER BY date ASC')
-    data = []
-    for coverage_entry in coverage_entries:
-      data.append({'date': coverage_entry.date,
-                   'line_coverage': coverage_entry.line_coverage,
-                   'function_coverage': coverage_entry.function_coverage,
-                  })
-
-    description = {
-        'date': ('datetime', 'Date'),
-        'line_coverage': ('number', 'Line Coverage'),
-        'function_coverage': ('number', 'Function Coverage')
-    }
-    coverage_data = gviz_api.DataTable(description, data)
-    coverage_json_data = coverage_data.ToJSon(order_by='date')
+    coverage_loader = load_coverage.CoverageDataLoader()
+    coverage_json_data = coverage_loader.load_coverage_json_data()
 
     # Fill in the template with the data and respond:
-    self.response.write(page_template % vars())
+    page_template_filename = 'templates/dashboard_template.html'
+    self.response.write(template.render(page_template_filename, vars()))
 
   def _show_error_page(self, error_message):
     self.response.write('<html><body>%s</body></html>' % error_message)

tools/quality_tracking/dashboard/gaeunit.py

@@ -0,0 +1 @@
+../../../third_party/gaeunit/gaeunit.py

tools/quality_tracking/dashboard/load_build_status.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+#  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+#
+#  Use of this source code is governed by a BSD-style license
+#  that can be found in the LICENSE file in the root of the source
+#  tree. An additional intellectual property rights grant can be found
+#  in the file PATENTS.  All contributing project authors may
+#  be found in the AUTHORS file in the root of the source tree.
+
+"""Loads build status data for the dashboard."""
+
+__author__ = 'phoglund@webrtc.org (Patrik Höglund)'
+
+from google.appengine.ext import db
+
+
+def _all_ok(statuses):
+  return filter(lambda status: status != "OK", statuses) == []
+
+
+def _get_first_entry(iterable):
+  if not iterable:
+    return None
+  for item in iterable:
+    return item
+
+
+class BuildStatusLoader:
+  """ Loads various build status data from the database."""
+
+  def load_build_status_data(self):
+    """Returns the latest conclusive build status for each bot.
+
+       The statuses OK or failed are considered to be conclusive.
+
+       The two most recent revisions are considered. The set of bots returned
+       will therefore be the bots that were reported the two most recent
+       revisions. This script will therefore adapt automatically to any changes
+       in the set of available bots.
+
+       Returns:
+           A list of BuildStatusData entities with one entity per bot.
+    """
+
+    build_status_entries = db.GqlQuery('SELECT * '
+                                       'FROM BuildStatusData '
+                                       'ORDER BY revision DESC ')
+
+    bots_to_latest_conclusive_entry = dict()
+    for entry in build_status_entries:
+      if entry.status == "building":
+        # The 'building' status it not conclusive, so discard this entry and
+        # pick up the entry for this bot on the next revision instead. That
+        # entry is guaranteed to have a status != 'building' since a bot cannot
+        # be building two revisions simultaneously.
+        continue
+      if bots_to_latest_conclusive_entry.has_key(entry.bot_name):
+        # We've already determined this bot's status.
+        continue
+
+      bots_to_latest_conclusive_entry[entry.bot_name] = entry
+
+    return bots_to_latest_conclusive_entry.values()
+
+  def load_last_modified_at(self):
+    build_status_root = db.GqlQuery('SELECT * '
+                                    'FROM BuildStatusRoot').get()
+    if not build_status_root:
+      # Operating on completely empty database
+      return None
+
+    return build_status_root.last_updated_at
+
+  def compute_lkgr(self):
+    """ Finds the most recent revision for which all bots are green.
+
+        Returns:
+            The last known good revision (as an integer) or None if there
+            is no green revision in the database.
+
+        Implementation note: The data store fetches stuff as we go, so we won't
+        read in the whole status table unless the LKGR is right at the end or
+        we don't have a LKGR.
+    """
+    build_status_entries = db.GqlQuery('SELECT * '
+                                       'FROM BuildStatusData '
+                                       'ORDER BY revision DESC ')
+
+    first_entry = _get_first_entry(build_status_entries)
+    if first_entry is None:
+      # No entries => no LKGR
+      return None
+
+    current_lkgr = first_entry.revision
+    statuses_for_current_lkgr = [first_entry.status]
+
+    for entry in build_status_entries:
+      if current_lkgr == entry.revision:
+        statuses_for_current_lkgr.append(entry.status)
+      else:
+        # Starting on new revision, check previous revision.
+        if _all_ok(statuses_for_current_lkgr):
+          # All bots are green; LKGR found.
+          return current_lkgr
+        else:
+          # Not all bots are green, so start over on the next revision.
+          current_lkgr = entry.revision
+          statuses_for_current_lkgr = [entry.status]
+
+    if _all_ok(statuses_for_current_lkgr):
+      # There was only one revision and it was OK.
+      return current_lkgr
+
+    # There is no all-green revision in the database.
+    return None

tools/quality_tracking/dashboard/load_coverage.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+#  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+#
+#  Use of this source code is governed by a BSD-style license
+#  that can be found in the LICENSE file in the root of the source
+#  tree. An additional intellectual property rights grant can be found
+#  in the file PATENTS.  All contributing project authors may
+#  be found in the AUTHORS file in the root of the source tree.
+
+"""Loads coverage data from the database."""
+
+__author__ = 'phoglund@webrtc.org (Patrik Höglund)'
+
+from google.appengine.ext import db
+import gviz_api
+
+
+class CoverageDataLoader:
+  """ Loads coverage data from the database."""
+
+  def load_coverage_json_data(self):
+    coverage_entries = db.GqlQuery('SELECT * '
+                                   'FROM CoverageData '
+                                   'ORDER BY date ASC')
+    data = []
+    for coverage_entry in coverage_entries:
+      data.append({'date': coverage_entry.date,
+                   'line_coverage': coverage_entry.line_coverage,
+                   'function_coverage': coverage_entry.function_coverage,
+                  })
+
+    description = {
+        'date': ('datetime', 'Date'),
+        'line_coverage': ('number', 'Line Coverage'),
+        'function_coverage': ('number', 'Function Coverage')
+    }
+    coverage_data = gviz_api.DataTable(description, data)
+    return coverage_data.ToJSon(order_by='date')

tools/quality_tracking/dashboard/stylesheets/stylesheet.css

@@ -0,0 +1,40 @@
+/********************************************************************
+*
+*  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+*
+*  Use of this source code is governed by a BSD-style license
+*  that can be found in the LICENSE file in the root of the source
+*  tree. An additional intellectual property rights grant can be found
+*  in the file PATENTS.  All contributing project authors may
+*  be found in the AUTHORS file in the root of the source tree.
+*
+*********************************************************************/
+
+.status_OK {
+  color: #FFFFFF;
+  background-color: #8fdf5f;
+}
+
+.status_failed {
+  color: #FFFFFF;
+  background-color: #e98080;
+}
+
+.status_building {
+  color: #666666;
+  background-color: #fffc6c;
+}
+
+.last_known_good_revision {
+  font-size: 800%;
+}
+
+.status_cell {
+  width: 100px;
+  text-align: center;
+}
+
+body {
+  margin-left: 35px;
+  margin-top: 25px;
+}

tools/quality_tracking/dashboard/templates/dashboard_template.html

@@ -1,6 +1,6 @@
 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
     "http://www.w3.org/TR/html4/strict.dtd">
-<html>  
+<html>
   <!--
   Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 
@@ -16,16 +16,24 @@
     <title>WebRTC Coverage Dashboard</title>
     <link href="http://code.google.com/css/codesite.pack.04102009.css"
           rel="stylesheet" type="text/css">
-    
+    <link href="stylesheets/stylesheet.css"
+          rel="stylesheet" type="text/css">
+
     <script src="https://www.google.com/jsapi" type="text/javascript"></script>
     <script type="text/javascript">
       google.load('visualization', '1', {packages:['table', 'corechart']});
 
       google.setOnLoadCallback(drawTable);
       function drawTable() {
-        /* Build data tables and views */      
+        /* Build data tables and views */
+        {% comment %}
+          Disable Django auto-escaping here since that will mess up our
+          coverage table JSON data otherwise.
+        {% endcomment %}
+        {% autoescape off %}
         var coverage_data_table =
-          new google.visualization.DataTable(%(coverage_json_data)s);
+          new google.visualization.DataTable({{ coverage_json_data }});
+        {% endautoescape %}
 
         /* Display tables and charts */
         var coverage_chart = new google.visualization.LineChart(
@@ -40,8 +48,38 @@
     </script>
   </head>
   <body>
-    <h1>WebRTC Dashboard</h1>
-    <h3>Coverage:</h3>
+
+    <h1>WebRTC Quality Dashboard</h1>
+    <h2>Current Build Status</h2>
+    <div>(as of {{ last_updated_at }})</div>
+    <table>
+      <tr>
+      {% for entry in build_status_data %}
+        <th class="status_cell">{{ entry.bot_name }}</th>
+      {% endfor %}
+      </tr>
+      <tr>
+      {% for entry in build_status_data %}
+        <td title="Last built revision {{ entry.revision }}"
+            class="status_cell status_{{entry.status}}">
+          {{entry.status}}
+        </td>
+      {% endfor %}
+      </tr>
+    </table>
+    <p></p>
+
+    <h2>Last Known Good Revision (LKGR)</h2>
+    <div class="last_known_good_revision">
+      {% if lkgr  %}
+        <a href="http://code.google.com/p/webrtc/source/detail?r={{ lkgr }}">
+          {{ lkgr }}</a>
+      {% else %}
+        ????
+      {% endif %}
+    </div>
+
+    <h2>Code Coverage History</h2>
     <div id="table_div_coverage"></div>
   </body>
 </html>

tools/quality_tracking/dashboard/test/load_build_status_test.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+#  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+#
+#  Use of this source code is governed by a BSD-style license
+#  that can be found in the LICENSE file in the root of the source
+#  tree. An additional intellectual property rights grant can be found
+#  in the file PATENTS.  All contributing project authors may
+#  be found in the AUTHORS file in the root of the source tree.
+
+__author__ = 'phoglund@webrtc.org (Patrik Höglund)'
+
+import unittest
+from google.appengine.ext import db
+from google.appengine.ext import testbed
+
+from add_build_status_data import BuildStatusData
+import load_build_status
+
+class LoadBuildStatusTest(unittest.TestCase):
+  def setUp(self):
+     # First, create an instance of the Testbed class.
+    self.testbed = testbed.Testbed()
+    # Then activate the testbed, which prepares the service stubs for use.
+    self.testbed.activate()
+    # Next, declare which service stubs you want to use.
+    self.testbed.init_datastore_v3_stub()
+
+  def test_returns_latest_nonbuilding_entries_when_loading_build_status(self):
+    BuildStatusData(bot_name="Bot1", revision=17,
+                    build_number=499, status="OK").put()
+    BuildStatusData(bot_name="Bot2", revision=17,
+                    build_number=505, status="OK").put()
+    BuildStatusData(bot_name="Bot3", revision=17,
+                    build_number=344, status="failed").put()
+    BuildStatusData(bot_name="Bot1", revision=18,
+                    build_number=499, status="building").put()
+    BuildStatusData(bot_name="Bot2", revision=18,
+                    build_number=505, status="failed").put()
+    BuildStatusData(bot_name="Bot3", revision=18,
+                    build_number=344, status="OK").put()
+
+    loader = load_build_status.BuildStatusLoader()
+    result = loader.load_build_status_data()
+
+    self.assertEqual(3, len(result))
+
+    # We make no guarantees on order, but we can use the fact that the testbed
+    # is deterministic to evaluate that the corrects bots were selected like so:
+    self.assertEqual("Bot1", result[0].bot_name)
+    self.assertEqual(17, result[0].revision)
+    self.assertEqual("OK", result[0].status)
+
+    self.assertEqual("Bot3", result[1].bot_name)
+    self.assertEqual(18, result[1].revision)
+    self.assertEqual("OK", result[1].status)
+
+    self.assertEqual("Bot2", result[2].bot_name)
+    self.assertEqual(18, result[2].revision)
+    self.assertEqual("failed", result[2].status)
+
+  def test_returns_lkgr_for_single_green_revision(self):
+    BuildStatusData(bot_name="Bot1", revision=17,
+                    build_number=499, status="OK").put()
+    BuildStatusData(bot_name="Bot2", revision=17,
+                    build_number=505, status="OK").put()
+    BuildStatusData(bot_name="Bot3", revision=17,
+                    build_number=344, status="OK").put()
+
+    loader = load_build_status.BuildStatusLoader()
+    self.assertEqual(17, loader.compute_lkgr())
+
+  def test_returns_correct_lkgr_with_most_recent_revision_failed(self):
+    BuildStatusData(bot_name="Bot1", revision=17,
+                    build_number=499, status="OK").put()
+    BuildStatusData(bot_name="Bot2", revision=17,
+                    build_number=505, status="OK").put()
+    BuildStatusData(bot_name="Bot3", revision=17,
+                    build_number=344, status="OK").put()
+    BuildStatusData(bot_name="Bot1", revision=18,
+                    build_number=499, status="OK").put()
+    BuildStatusData(bot_name="Bot2", revision=18,
+                    build_number=505, status="failed").put()
+    BuildStatusData(bot_name="Bot3", revision=18,
+                    build_number=344, status="OK").put()
+
+    loader = load_build_status.BuildStatusLoader()
+    self.assertEqual(17, loader.compute_lkgr())
+
+  def test_returns_none_if_no_revisions(self):
+    loader = load_build_status.BuildStatusLoader()
+    self.assertEqual(None, loader.compute_lkgr())
+
+  def test_returns_none_if_no_green_revisions(self):
+    BuildStatusData(bot_name="Bot2", revision=18,
+                    build_number=505, status="failed").put()
+
+    loader = load_build_status.BuildStatusLoader()
+    self.assertEqual(None, loader.compute_lkgr())
+
+  def test_skips_partially_building_revisions(self):
+    BuildStatusData(bot_name="Bot1", revision=18,
+                    build_number=499, status="building").put()
+    BuildStatusData(bot_name="Bot2", revision=18,
+                    build_number=505, status="OK").put()
+    BuildStatusData(bot_name="Bot1", revision=17,
+                    build_number=344, status="OK").put()
+
+    loader = load_build_status.BuildStatusLoader()
+    self.assertEqual(17, loader.compute_lkgr())
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/quality_tracking/dashboard_connection.py

@@ -67,17 +67,18 @@ class DashboardConnection:
        make the OAuth request. These concepts are described in the class
        description.
 
+       The server is expected to respond with HTTP status 200 and a completely
+       empty response if the call failed. The server may put diagnostic
+       information in the response.
+
        Args:
            sub_url: A relative url within the dashboard domain, for example
                /add_coverage_data.
            parameters: A dict which maps from POST parameter names to values.
 
-       Returns:
-           A httplib response object.
-
        Raises:
            FailedToReportToDashboard: If the dashboard didn't respond
-               with HTTP 200 to our request.
+               with HTTP 200 to our request or if the response is non-empty.
     """
     consumer = oauth.OAuthConsumer(self.consumer_key_, self.consumer_secret_)
     create_oauth_request = oauth.OAuthRequest.from_consumer_and_token
@@ -101,12 +102,17 @@ class DashboardConnection:
     connection.close()
 
     if response.status != 200:
-      message = ('Error: Failed to report to %s%s: got response %d (%s)' %
+      message = ('Failed to report to %s%s: got response %d (%s)' %
                  (constants.DASHBOARD_SERVER, sub_url, response.status,
                   response.reason))
       raise FailedToReportToDashboard(message)
 
-    return response
+    # The response content should be empty on success, so check that:
+    response_content = response.read()
+    if response_content:
+      message = ('Dashboard reported the following error: %s.' %
+                 response_content)
+      raise FailedToReportToDashboard(message)
 
   def _read_access_token(self, filename):
     input_file = shelve.open(filename)

tools/quality_tracking/request_oauth_permission.py

@@ -36,6 +36,7 @@ import oauth2 as oauth
 
 import constants
 
+
 class FailedToRequestPermissionException(Exception):
   pass
 

tools/quality_tracking/tgrid_parser.py

@@ -66,7 +66,7 @@ def parse_tgrid_page(html):
     result.update(_parse_builds(revision, builds_for_revision_html))
 
   if not result:
-   raise FailedToParseBuildStatus('Could not find any build statuses in %s.' %
-                                  html)
+    raise FailedToParseBuildStatus('Could not find any build statuses in %s.' %
+                                   html)
 
   return result

tools/quality_tracking/track_build_status.py

@@ -16,7 +16,6 @@ __author__ = 'phoglund@webrtc.org (Patrik Höglund)'
 
 
 import httplib
-import re
 
 import constants
 import dashboard_connection
@@ -50,10 +49,9 @@ def _main():
 
   bot_to_status_mapping = _download_and_parse_build_status()
 
-  response = dashboard.send_post_request(constants.ADD_BUILD_STATUS_DATA_URL,
-                                         bot_to_status_mapping)
+  dashboard.send_post_request(constants.ADD_BUILD_STATUS_DATA_URL,
+                              bot_to_status_mapping)
 
-  print response.read()
 
 if __name__ == '__main__':
   _main()

tools/quality_tracking/track_coverage.py

@@ -28,7 +28,6 @@ __author__ = 'phoglund@webrtc.org (Patrik Höglund)'
 
 import os
 import re
-import sys
 import time
 
 import constants
@@ -90,15 +89,7 @@ def _report_coverage_to_dashboard(dashboard, now, line_coverage,
                 'function_coverage': '%f' % function_coverage
                }
 
-  response = dashboard.send_post_request(constants.ADD_COVERAGE_DATA_URL,
-                                         parameters)
-
-  # The response content should be empty on success, so check that:
-  response_content = response.read()
-  if response_content:
-    message = ('Error: Dashboard reported the following error: %s.' %
-               response_content)
-    raise dashboard_connection.FailedToReportToDashboard(message)
+  dashboard.send_post_request(constants.ADD_COVERAGE_DATA_URL, parameters)
 
 
 def _main():