generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Implemented branch coverage and integration bot coverage on the dashboard.

Browse Source 
BUG=
TEST=

Review URL: https://webrtc-codereview.appspot.com/434002

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1873 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
phoglund@webrtc.org 2012-03-12 09:12:32 +00:00
parent 2e34c88c37
commit fc402760e9
5 changed files with 142 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
tools/quality_tracking/dashboard/add_coverage_data.py
View File

@ -12,25 +12,29 @@
__author__ = 'phoglund@webrtc.org (Patrik Höglund)'
import datetime
from datetime import datetime
import logging
from google.appengine.ext import db
import oauth_post_request_handler
REPORT_CATEGORIES = ('small_medium_tests', 'large_tests')
class CoverageData(db.Model):
"""This represents one coverage report from the build bot."""
# The date the report was made.
date = db.DateTimeProperty(required=True)
# Coverage percentages.
line_coverage = db.FloatProperty(required=True)
function_coverage = db.FloatProperty(required=True)
branch_coverage = db.FloatProperty()
def _parse_percentage(string_value):
percentage = float(string_value)
if percentage < 0.0 or percentage > 100.0:
raise ValueError('%s is not a valid percentage.' % string_value)
return percentage
# The report category must be one of the REPORT_CATEGORIES.
report_category = db.CategoryProperty()
class AddCoverageData(oauth_post_request_handler.OAuthPostRequestHandler):
@ -40,19 +44,24 @@ class AddCoverageData(oauth_post_request_handler.OAuthPostRequestHandler):
the regular oauth_* parameters, these values:
date: The POSIX timestamp for when the coverage observation was made.
line_coverage: A float percentage in the interval 0-100.0.
function_coverage: A float percentage in the interval 0-100.0.
report_category: A value in REPORT_CATEGORIES which characterizes the
coverage information (e.g. is the coverage from small / medium tests
or large tests?)
line_coverage: Line coverage percentage.
function_coverage: Function coverage percentage.
branch_coverage: Branch coverage percentage.
"""
def _parse_and_store_data(self):
try:
posix_time = int(self.request.get('date'))
parsed_date = datetime.datetime.fromtimestamp(posix_time)
request_posix_timestamp = float(self.request.get('oauth_timestamp'))
parsed_date = datetime.fromtimestamp(request_posix_timestamp)
line_coverage_string = self.request.get('line_coverage')
line_coverage = _parse_percentage(line_coverage_string)
function_coverage_string = self.request.get('function_coverage')
function_coverage = _parse_percentage(function_coverage_string)
line_coverage = self._parse_percentage('line_coverage')
function_coverage = self._parse_percentage('function_coverage')
branch_coverage = self._parse_percentage('branch_coverage')
report_category = self._parse_category('report_category')
except ValueError as error:
logging.warn('Invalid parameter in request: %s.' % error)
@ -61,6 +70,21 @@ class AddCoverageData(oauth_post_request_handler.OAuthPostRequestHandler):
item = CoverageData(date=parsed_date,
line_coverage=line_coverage,
function_coverage=function_coverage)
function_coverage=function_coverage,
branch_coverage=branch_coverage,
report_category=report_category)
item.put()
def _parse_percentage(self, key):
"""Parses out a percentage value from the request."""
value = float(self.request.get(key))
if percentage < 0.0 or percentage > 100.0:
raise ValueError('%s is not a valid percentage.' % string_value)
return percentage
def _parse_category(self, key):
value = self.request.get(key)
if value in REPORT_CATEGORIES:
return value
else:
raise ValueError("Invalid category %s." % value)

5
tools/quality_tracking/dashboard/dashboard.py
View File

@ -40,7 +40,10 @@ class ShowDashboard(webapp2.RequestHandler):
lkgr = build_status_loader.compute_lkgr()
coverage_loader = load_coverage.CoverageDataLoader()
coverage_json_data = coverage_loader.load_coverage_json_data()
small_medium_coverage_json_data = (
coverage_loader.load_coverage_json_data('small_medium_tests'))
large_coverage_json_data = (
coverage_loader.load_coverage_json_data('large_tests'))
page_template_filename = 'templates/dashboard_template.html'
self.response.write(template.render(page_template_filename, vars()))

18
tools/quality_tracking/dashboard/load_coverage.py
View File

@ -12,6 +12,8 @@
__author__ = 'phoglund@webrtc.org (Patrik Höglund)'
import logging
from google.appengine.ext import db
import gviz_api
@ -19,21 +21,27 @@ import gviz_api
class CoverageDataLoader:
""" Loads coverage data from the database."""
def load_coverage_json_data(self):
def load_coverage_json_data(self, report_category):
coverage_entries = db.GqlQuery('SELECT * '
'FROM CoverageData '
'ORDER BY date ASC')
'WHERE report_category = :1 '
'ORDER BY date ASC', report_category)
data = []
for coverage_entry in coverage_entries:
data.append({'date': coverage_entry.date,
# Note: The date column must be first in alphabetical order since it is
# the primary column. This is a bug in the gviz api (or at least it
# doesn't make much sense).
data.append({'aa_date': coverage_entry.date,
'line_coverage': coverage_entry.line_coverage,
'function_coverage': coverage_entry.function_coverage,
'branch_coverage': coverage_entry.branch_coverage,
})
description = {
'date': ('datetime', 'Date'),
'aa_date': ('datetime', 'Date'),
'line_coverage': ('number', 'Line Coverage'),
'function_coverage': ('number', 'Function Coverage')
'function_coverage': ('number', 'Function Coverage'),
'branch_coverage': ('number', 'Branch Coverage'),
}
coverage_data = gviz_api.DataTable(description, data)
return coverage_data.ToJSon(order_by='date')

34
tools/quality_tracking/dashboard/templates/dashboard_template.html
View File

@ -32,16 +32,29 @@
coverage table JSON data otherwise.
{% endcomment %}
{% autoescape off %}
var coverage_data_table =
new google.visualization.DataTable({{ coverage_json_data }});
var small_medium_coverage_data_table =
new google.visualization.DataTable(
{{ small_medium_coverage_json_data }});
var large_coverage_data_table =
new google.visualization.DataTable(
{{ large_coverage_json_data }});
{% endautoescape %}
/* Display tables and charts */
var coverage_chart = new google.visualization.LineChart(
document.getElementById('table_div_coverage'));
coverage_chart.draw(coverage_data_table, {
colors: ['blue', 'red'],
vAxis: {title: 'Coverage'},
var small_medium_coverage_chart = new google.visualization.LineChart(
document.getElementById('table_div_small_medium_coverage'));
small_medium_coverage_chart.draw(small_medium_coverage_data_table, {
colors: ['blue', 'red', 'black'],
vAxis: {title: 'Coverage (%)'},
hAxis: {title: 'Date'},
width: 1200, height: 300,
});
var large_coverage_chart = new google.visualization.LineChart(
document.getElementById('table_div_large_coverage'));
large_coverage_chart.draw(large_coverage_data_table, {
colors: ['blue', 'red', 'black'],
vAxis: {title: 'Coverage (%)'},
hAxis: {title: 'Date'},
width: 1200, height: 300,
});
@ -49,7 +62,6 @@
</script>
</head>
<body>
<h1>WebRTC Quality Dashboard</h1>
<h2>Current Build Status</h2>
<div>(as of {{ last_updated_at }} UTC)</div>
@ -80,7 +92,9 @@
{% endif %}
</div>
<h2>Code Coverage History</h2>
<div id="table_div_coverage"></div>
<h2>Code Coverage History (Small / Medium Tests)</h2>
<div id="table_div_small_medium_coverage"></div>
<h2>Code Coverage History (Large Tests)</h2>
<div id="table_div_large_coverage"></div>
</body>
</html>

84
tools/quality_tracking/track_coverage.py
View File

@ -28,6 +28,7 @@ __author__ = 'phoglund@webrtc.org (Patrik Höglund)'
import os
import re
import sys
import time
import constants
@ -42,28 +43,39 @@ class CouldNotFindCoverageDirectory(Exception):
pass
def _find_latest_32bit_debug_build(www_directory_contents, coverage_www_dir):
"""Finds the latest 32-bit coverage directory in the directory listing.
def _find_latest_build_coverage(www_directory_contents, coverage_www_dir,
directory_prefix):
"""Finds the most recent coverage directory in the directory listing.
Coverage directories have the form Linux32bitDBG_<number>. There may be
other directories in the list though, for instance for other build
configurations. We assume here that build numbers keep rising and never
wrap around or anything like that.
We assume here that build numbers keep rising and never wrap around.
Args:
www_directory_contents: A list of entries in the coverage directory.
coverage_www_dir: The coverage directory on the bot.
directory_prefix: Coverage directories have the form <prefix><number>,
and the prefix is different on different bots. The prefix is
generally the builder name, such as Linux32DBG.
Returns:
The most recent directory name.
Raises:
CouldNotFindCoverageDirectory: if we failed to find coverage data.
"""
found_build_numbers = []
for entry in www_directory_contents:
match = re.match('Linux32DBG_(\d+)', entry)
match = re.match(directory_prefix + '(\d+)', entry)
if match is not None:
found_build_numbers.append(int(match.group(1)))
if not found_build_numbers:
raise CouldNotFindCoverageDirectory('Error: Found no 32-bit '
'debug build in directory %s.' %
coverage_www_dir)
raise CouldNotFindCoverageDirectory('Error: Found no directories %s* '
'in directory %s.' %
(directory_prefix, coverage_www_dir))
most_recent = max(found_build_numbers)
return 'Linux32DBG_' + str(most_recent)
return directory_prefix + str(most_recent)
def _grab_coverage_percentage(label, index_html_contents):
@ -84,25 +96,37 @@ def _grab_coverage_percentage(label, index_html_contents):
raise FailedToParseCoverageHtml('%s is not a float.' % match.group(1))
def _report_coverage_to_dashboard(dashboard, now, line_coverage,
function_coverage):
parameters = {'date': '%d' % now,
'line_coverage': '%f' % line_coverage,
'function_coverage': '%f' % function_coverage
def _report_coverage_to_dashboard(dashboard, line_coverage, function_coverage,
branch_coverage, report_category):
parameters = {'line_coverage': '%f' % line_coverage,
'function_coverage': '%f' % function_coverage,
'branch_coverage': '%f' % branch_coverage,
'report_category': report_category,
}
dashboard.send_post_request(constants.ADD_COVERAGE_DATA_URL, parameters)
def _main():
def _main(report_category, directory_prefix):
"""Grabs coverage data from disk on a bot and publishes it.
Args:
report_category: The kind of coverage to report. The dashboard
application decides what is acceptable here (see
dashboard/add_coverage_data.py for more information).
directory_prefix: This bot's coverage directory prefix. Generally a bot's
coverage directories will have the form <prefix><build number>,
like Linux32DBG_345.
"""
dashboard = dashboard_connection.DashboardConnection(constants.CONSUMER_KEY)
dashboard.read_required_files(constants.CONSUMER_SECRET_FILE,
constants.ACCESS_TOKEN_FILE)
coverage_www_dir = constants.BUILD_BOT_COVERAGE_WWW_DIRECTORY
www_dir_contents = os.listdir(coverage_www_dir)
latest_build_directory = _find_latest_32bit_debug_build(www_dir_contents,
coverage_www_dir)
latest_build_directory = _find_latest_build_coverage(www_dir_contents,
coverage_www_dir,
directory_prefix)
index_html_path = os.path.join(coverage_www_dir, latest_build_directory,
'index.html')
@ -111,12 +135,26 @@ def _main():
line_coverage = _grab_coverage_percentage('Lines:', whole_file)
function_coverage = _grab_coverage_percentage('Functions:', whole_file)
now = int(time.time())
branch_coverage = _grab_coverage_percentage('Branches:', whole_file)
_report_coverage_to_dashboard(dashboard, now, line_coverage,
function_coverage)
_report_coverage_to_dashboard(dashboard, line_coverage, function_coverage,
branch_coverage, report_category)
def _parse_args():
if len(sys.argv) != 3:
print ('Usage: %s <coverage category> <directory prefix>\n\n'
'The coverage category describes the kind of coverage you are '
'uploading. Known acceptable values are small_medium_tests and'
'large_tests. The directory prefix is what the directories in %s '
'are prefixed on this bot (such as Linux32DBG_).' %
(sys.argv[0], constants.BUILD_BOT_COVERAGE_WWW_DIRECTORY))
return (None, None)
return (sys.argv[1], sys.argv[2])
if __name__ == '__main__':
_main()
report_category, directory_prefix = _parse_args()
if report_category:
_main(report_category, directory_prefix)