diff --git a/tools/DEPS b/tools/DEPS index 8c7429c40..7c94d637f 100644 --- a/tools/DEPS +++ b/tools/DEPS @@ -3,14 +3,6 @@ # proxies. deps = { - # Used by quality_tracking. - "tools/third_party/gaeunit": - "http://code.google.com/p/gaeunit.git@e16d5bd4", - - # Used by quality_tracking. - "tools/third_party/oauth2": - "http://github.com/simplegeo/python-oauth2.git@a83f4a29", - # Used by tools/quality_tracking/dashboard and tools/python_charts. "tools/third_party/google-visualization-python": "http://google-visualization-python.googlecode.com/svn/trunk/@15", diff --git a/tools/quality_tracking/OWNERS b/tools/quality_tracking/OWNERS deleted file mode 100644 index 323e8e72d..000000000 --- a/tools/quality_tracking/OWNERS +++ /dev/null @@ -1,2 +0,0 @@ -phoglund@webrtc.org -kjellander@webrtc.org diff --git a/tools/quality_tracking/README b/tools/quality_tracking/README deleted file mode 100644 index faf3e7a85..000000000 --- a/tools/quality_tracking/README +++ /dev/null @@ -1,31 +0,0 @@ -This file describes the coverage tracking script and the coverage dashboard. - -ABSTRACT: -The intention of this small tracking system is to track code coverage data -over time. Since code coverage is continuously recomputed on the build bots, -the track_coverage.py script is intended to run on the build bot as a cron job -and extract the data from there. The dashboard doesn't care how often this -script runs, but running each hour should be more than enough. - -The track_coverage.py script uses OAuth to authenticate itself. In order to do -this, it needs two files: consumer.secret and access.token. The consumer secret -is known within the organization and is stored in a plain file on the bot -running the scripts (we don't want to check in this secret in the code in the -public repository). The consumer secret is a plain file with a single line -containing the secret string. - -The access.token file is generated by request_oauth_permission.py. It does this -by going through the three-legged OAuth authorization process. An administrator -of the dashboard must approve the request from the script. Once that is done, -access.token will be written and track_coverage.py will be able to report -results. - -HOW TO RUN LOCALLY: -Follow the following instructions: -http://code.google.com/appengine/docs/python/gettingstartedpython27/devenvironment.html -The dashboard can be started on 127.0.0.1:8080 using the dev_appserver.py script -as described in the above URL (and in the following 'hello world' page). - -HOW TO DEPLOY: -Follow the following instructions: -http://code.google.com/appengine/docs/python/gettingstartedpython27/uploading.html \ No newline at end of file diff --git a/tools/quality_tracking/constants.py b/tools/quality_tracking/constants.py deleted file mode 100644 index 32c8a164b..000000000 --- a/tools/quality_tracking/constants.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -#-*- coding: utf-8 -*- -# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. -# -# Use of this source code is governed by a BSD-style license -# that can be found in the LICENSE file in the root of the source -# tree. An additional intellectual property rights grant can be found -# in the file PATENTS. All contributing project authors may -# be found in the AUTHORS file in the root of the source tree. - -"""Contains tweakable constants for quality dashboard utility scripts.""" - -# This identifies our application using the information we got when we -# registered the application on Google appengine. -DASHBOARD_SERVER = 'webrtc-dashboard.appspot.com' -DASHBOARD_SERVER_HTTP = 'http://' + DASHBOARD_SERVER -CONSUMER_KEY = DASHBOARD_SERVER -CONSUMER_SECRET_FILE = 'consumer.secret' -ACCESS_TOKEN_FILE = 'access.token' - -# OAuth URL:s. -REQUEST_TOKEN_URL = DASHBOARD_SERVER_HTTP + '/_ah/OAuthGetRequestToken' -AUTHORIZE_TOKEN_URL = DASHBOARD_SERVER_HTTP + '/_ah/OAuthAuthorizeToken' -ACCESS_TOKEN_URL = DASHBOARD_SERVER_HTTP + '/_ah/OAuthGetAccessToken' - -# Build bot constants. -BUILD_BOT_COVERAGE_WWW_DIRECTORY = '/var/www/coverage' - -# Dashboard data input URLs. -ADD_COVERAGE_DATA_URL = DASHBOARD_SERVER_HTTP + '/add_coverage_data' diff --git a/tools/quality_tracking/dashboard/add_coverage_data.py b/tools/quality_tracking/dashboard/add_coverage_data.py deleted file mode 100644 index ed3abad70..000000000 --- a/tools/quality_tracking/dashboard/add_coverage_data.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python -#-*- coding: utf-8 -*- -# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. -# -# Use of this source code is governed by a BSD-style license -# that can be found in the LICENSE file in the root of the source -# tree. An additional intellectual property rights grant can be found -# in the file PATENTS. All contributing project authors may -# be found in the AUTHORS file in the root of the source tree. - -"""Implements a handler for adding coverage data.""" - -from datetime import datetime -import logging - -from google.appengine.ext import db - -import oauth_post_request_handler - -REPORT_CATEGORIES = ('small_medium_tests', 'large_tests') - - -class CoverageData(db.Model): - """This represents one coverage report from the build bot.""" - # The date the report was made. - date = db.DateTimeProperty(required=True) - - # Coverage percentages. - line_coverage = db.FloatProperty(required=True) - function_coverage = db.FloatProperty(required=True) - branch_coverage = db.FloatProperty() - - # The report category must be one of the REPORT_CATEGORIES. - report_category = db.CategoryProperty() - - -class AddCoverageData(oauth_post_request_handler.OAuthPostRequestHandler): - """Used to report coverage data. - - Coverage data is reported as a POST request and should contain, aside from - the regular oauth_* parameters, these values: - - date: The POSIX timestamp for when the coverage observation was made. - report_category: A value in REPORT_CATEGORIES which characterizes the - coverage information (e.g. is the coverage from small / medium tests - or large tests?) - - line_coverage: Line coverage percentage. - function_coverage: Function coverage percentage. - branch_coverage: Branch coverage percentage. - """ - def _parse_and_store_data(self): - try: - request_posix_timestamp = float(self.request.get('oauth_timestamp')) - parsed_date = datetime.fromtimestamp(request_posix_timestamp) - - line_coverage = self._parse_percentage('line_coverage') - function_coverage = self._parse_percentage('function_coverage') - branch_coverage = self._parse_percentage('branch_coverage') - report_category = self._parse_category('report_category') - - except ValueError as error: - logging.warn('Invalid parameter in request: %s.' % error) - self.response.set_status(400) - return - - item = CoverageData(date=parsed_date, - line_coverage=line_coverage, - function_coverage=function_coverage, - branch_coverage=branch_coverage, - report_category=report_category) - item.put() - - def _parse_percentage(self, key): - """Parses out a percentage value from the request.""" - string_value = self.request.get(key) - percentage = float(string_value) - if percentage < 0.0 or percentage > 100.0: - raise ValueError('%s is not a valid percentage.' % string_value) - return percentage - - def _parse_category(self, key): - value = self.request.get(key) - if value in REPORT_CATEGORIES: - return value - else: - raise ValueError("Invalid category %s." % value) diff --git a/tools/quality_tracking/dashboard/app.yaml b/tools/quality_tracking/dashboard/app.yaml deleted file mode 100644 index 1d6169d90..000000000 --- a/tools/quality_tracking/dashboard/app.yaml +++ /dev/null @@ -1,32 +0,0 @@ -application: webrtc-dashboard -version: 1 -runtime: python27 -api_version: 1 -threadsafe: false - -handlers: -# Serve stylesheets, perf dashboard, and images statically. -- url: /stylesheets - static_dir: stylesheets -- url: /perf - static_dir: static -- url: /images - static_dir: static -- url: /lkgr - static_files: static/lkgr_redirect.html - upload: static/lkgr_redirect.html - -# This magic file is here to prove to the Google Account Domain Management -# that we own this domain. It needs to stay there so the domain management -# doesn't get suspicious. -- url: /google403c95edcde16425.html - static_files: static/google403c95edcde16425.html - upload: static/google403c95edcde16425.html - -# Note: tests should be disabled in production. -# - url: /test.* -# script: gaeunit.py - -# Redirect all other requests to our dynamic handlers. -- url: /.* - script: main.app diff --git a/tools/quality_tracking/dashboard/dashboard.py b/tools/quality_tracking/dashboard/dashboard.py deleted file mode 100644 index 432501104..000000000 --- a/tools/quality_tracking/dashboard/dashboard.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python -#-*- coding: utf-8 -*- -# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. -# -# Use of this source code is governed by a BSD-style license -# that can be found in the LICENSE file in the root of the source -# tree. An additional intellectual property rights grant can be found -# in the file PATENTS. All contributing project authors may -# be found in the AUTHORS file in the root of the source tree. - -"""Implements the quality tracker dashboard and reporting facilities.""" - -from google.appengine.ext.webapp import template -import webapp2 - -import load_coverage - - -class ShowDashboard(webapp2.RequestHandler): - """Shows the dashboard page. - - The page is shown by grabbing data we have stored previously - in the App Engine database using the AddCoverageData handler. - """ - def get(self): - coverage_loader = load_coverage.CoverageDataLoader() - # pylint: disable=W0612 - small_medium_coverage_json_data = ( - coverage_loader.load_coverage_json_data('small_medium_tests')) - # pylint: disable=W0612 - large_coverage_json_data = ( - coverage_loader.load_coverage_json_data('large_tests')) - - page_template_filename = 'templates/dashboard_template.html' - self.response.write(template.render(page_template_filename, vars())) - - def _show_error_page(self, error_message): - self.response.write('%s' % error_message) - diff --git a/tools/quality_tracking/dashboard/gaeunit.py b/tools/quality_tracking/dashboard/gaeunit.py deleted file mode 120000 index a93f6bd00..000000000 --- a/tools/quality_tracking/dashboard/gaeunit.py +++ /dev/null @@ -1 +0,0 @@ -../../../third_party/gaeunit/gaeunit.py \ No newline at end of file diff --git a/tools/quality_tracking/dashboard/gviz_api.py b/tools/quality_tracking/dashboard/gviz_api.py deleted file mode 120000 index c9dca90fa..000000000 --- a/tools/quality_tracking/dashboard/gviz_api.py +++ /dev/null @@ -1 +0,0 @@ -../../third_party/google-visualization-python/gviz_api.py \ No newline at end of file diff --git a/tools/quality_tracking/dashboard/index.yaml b/tools/quality_tracking/dashboard/index.yaml deleted file mode 100644 index 7a20411e9..000000000 --- a/tools/quality_tracking/dashboard/index.yaml +++ /dev/null @@ -1,16 +0,0 @@ -indexes: - -# AUTOGENERATED - -# This index.yaml is automatically updated whenever the dev_appserver -# detects that a new type of query is run. If you want to manage the -# index.yaml file manually, remove the above marker line (the line -# saying "# AUTOGENERATED"). If you want to manage some indexes -# manually, move them above the marker line. The index.yaml file is -# automatically uploaded to the admin console when you next deploy -# your application using appcfg.py. - -- kind: CoverageData - properties: - - name: report_category - - name: date diff --git a/tools/quality_tracking/dashboard/load_coverage.py b/tools/quality_tracking/dashboard/load_coverage.py deleted file mode 100644 index a02b7338c..000000000 --- a/tools/quality_tracking/dashboard/load_coverage.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python -#-*- coding: utf-8 -*- -# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. -# -# Use of this source code is governed by a BSD-style license -# that can be found in the LICENSE file in the root of the source -# tree. An additional intellectual property rights grant can be found -# in the file PATENTS. All contributing project authors may -# be found in the AUTHORS file in the root of the source tree. - -"""Loads coverage data from the database.""" - -from google.appengine.ext import db -import gviz_api - - -class CoverageDataLoader: - """ Loads coverage data from the database.""" - @staticmethod - def load_coverage_json_data(report_category): - coverage_entries = db.GqlQuery('SELECT * ' - 'FROM CoverageData ' - 'WHERE report_category = :1 ' - 'ORDER BY date ASC', report_category) - data = [] - for coverage_entry in coverage_entries: - # Note: The date column must be first in alphabetical order since it is - # the primary column. This is a bug in the gviz api (or at least it - # doesn't make much sense). - data.append({'aa_date': coverage_entry.date, - 'line_coverage': coverage_entry.line_coverage, - 'function_coverage': coverage_entry.function_coverage, - }) - - description = { - 'aa_date': ('datetime', 'Date'), - 'line_coverage': ('number', 'Line Coverage'), - 'function_coverage': ('number', 'Function Coverage'), - } - coverage_data = gviz_api.DataTable(description, data) - return coverage_data.ToJSon(order_by='date') diff --git a/tools/quality_tracking/dashboard/main.py b/tools/quality_tracking/dashboard/main.py deleted file mode 100644 index ead6a9f7a..000000000 --- a/tools/quality_tracking/dashboard/main.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python -#-*- coding: utf-8 -*- -# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. -# -# Use of this source code is governed by a BSD-style license -# that can be found in the LICENSE file in the root of the source -# tree. An additional intellectual property rights grant can be found -# in the file PATENTS. All contributing project authors may -# be found in the AUTHORS file in the root of the source tree. - -"""Connects all URLs with their respective handlers.""" - -import webapp2 - -import add_coverage_data -import dashboard - -app = webapp2.WSGIApplication([('/', dashboard.ShowDashboard), - ('/add_coverage_data', - add_coverage_data.AddCoverageData)], - debug=True) diff --git a/tools/quality_tracking/dashboard/oauth_post_request_handler.py b/tools/quality_tracking/dashboard/oauth_post_request_handler.py deleted file mode 100644 index 0dc40bfb3..000000000 --- a/tools/quality_tracking/dashboard/oauth_post_request_handler.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python -#-*- coding: utf-8 -*- -# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. -# -# Use of this source code is governed by a BSD-style license -# that can be found in the LICENSE file in the root of the source -# tree. An additional intellectual property rights grant can be found -# in the file PATENTS. All contributing project authors may -# be found in the AUTHORS file in the root of the source tree. - -"""Provides a OAuth request handler base class.""" - -from google.appengine.api import oauth -import logging -import webapp2 - - -class UserNotAuthenticatedException(Exception): - """Gets thrown if a user is not permitted to store data.""" - pass - - -class OAuthPostRequestHandler(webapp2.RequestHandler): - """Works like a normal request handler but adds OAuth authentication. - - This handler will expect a proper OAuth request over POST. This abstract - class deals with the authentication but leaves user-defined data handling - to its subclasses. Subclasses should not implement the post() method but - the _parse_and_store_data() method. Otherwise they may act like regular - request handlers. Subclasses should NOT override the get() method. - - The handler will accept an OAuth request if it is correctly formed and - the consumer is acting on behalf of an administrator for the dashboard. - """ - def __init__(self, request=None, response=None): - webapp2.RequestHandler.__init__(self, request, response) - - def post(self): - try: - self._authenticate_user() - except UserNotAuthenticatedException as exception: - logging.warn('Failed to authenticate: %s.' % exception) - self.response.set_status(403) - return - - # Do the actual work. - self._parse_and_store_data() - - def _parse_and_store_data(self): - """Reads data from POST request and responds accordingly.""" - raise NotImplementedError('You must override this method!') - - @staticmethod - def _authenticate_user(): - try: - if oauth.is_current_user_admin(): - # The user on whose behalf we are acting is indeed an administrator - # of this application, so we're good to go. - logging.info('Authenticated on behalf of user %s.' % - oauth.get_current_user()) - return - else: - raise UserNotAuthenticatedException('We are acting on behalf of ' - 'user %s, but that user is not ' - 'an administrator.' % - oauth.get_current_user()) - except oauth.OAuthRequestError as exception: - raise UserNotAuthenticatedException('Invalid OAuth request: %s' % - exception.__class__.__name__) diff --git a/tools/quality_tracking/dashboard/static/audio_perf.html b/tools/quality_tracking/dashboard/static/audio_perf.html deleted file mode 100644 index 1309eb7d7..000000000 --- a/tools/quality_tracking/dashboard/static/audio_perf.html +++ /dev/null @@ -1,15 +0,0 @@ - - - WebRTC Performance Metrics - Audio Quality - - - -

Audio Quality WebAudio -> PeerConnection call (PESQ) Linux/Win

- - -

Audio Quality Voice Engine E2E test (PESQ) Linux

- -

Audio Processing time per 10 ms frame (ms) Linux

- - - \ No newline at end of file diff --git a/tools/quality_tracking/dashboard/static/audio_perf_chrome.html b/tools/quality_tracking/dashboard/static/audio_perf_chrome.html deleted file mode 100644 index a05f2c85f..000000000 --- a/tools/quality_tracking/dashboard/static/audio_perf_chrome.html +++ /dev/null @@ -1,24 +0,0 @@ - - - WebRTC Performance Metrics - Audio Quality Chrome - - - -

Chrome Audio: WebRTC Loopback With Signal Processing (ms) Linux/Win/Mac

- - - -

Chrome Audio: WebRTC Loopback Without Signal Processing (ms) Linux/Win/Mac

- - - -

Chrome Audio: WebRTC playout setup time (ms) Linux/Win/Mac

- - - -

Chrome Audio: WebRTC recording setup time (ms) Linux/Win/Mac

- - - - - \ No newline at end of file diff --git a/tools/quality_tracking/dashboard/static/google403c95edcde16425.html b/tools/quality_tracking/dashboard/static/google403c95edcde16425.html deleted file mode 100644 index 95c7e2d55..000000000 --- a/tools/quality_tracking/dashboard/static/google403c95edcde16425.html +++ /dev/null @@ -1 +0,0 @@ -google-site-verification: google403c95edcde16425.html \ No newline at end of file diff --git a/tools/quality_tracking/dashboard/static/index.html b/tools/quality_tracking/dashboard/static/index.html deleted file mode 100644 index 2b18c0eff..000000000 --- a/tools/quality_tracking/dashboard/static/index.html +++ /dev/null @@ -1,5 +0,0 @@ - - - - - \ No newline at end of file diff --git a/tools/quality_tracking/dashboard/static/lkgr_redirect.html b/tools/quality_tracking/dashboard/static/lkgr_redirect.html deleted file mode 100644 index 49a473390..000000000 --- a/tools/quality_tracking/dashboard/static/lkgr_redirect.html +++ /dev/null @@ -1,6 +0,0 @@ - - - - - \ No newline at end of file diff --git a/tools/quality_tracking/dashboard/static/video_perf.html b/tools/quality_tracking/dashboard/static/video_perf.html deleted file mode 100644 index d3bef66b9..000000000 --- a/tools/quality_tracking/dashboard/static/video_perf.html +++ /dev/null @@ -1,20 +0,0 @@ - - - WebRTC Performance Metrics - Video Quality - - - -

Video Quality (PSNR) Linux/Win/Mac

- - - -

Video Quality (SSIM) Linux/Win/Mac

- - - -

Video Quality (Unique frame count) Linux/Win/Mac

- - - - - \ No newline at end of file diff --git a/tools/quality_tracking/dashboard/static/vie_auto_test_perf.html b/tools/quality_tracking/dashboard/static/vie_auto_test_perf.html deleted file mode 100644 index 1828bd8ee..000000000 --- a/tools/quality_tracking/dashboard/static/vie_auto_test_perf.html +++ /dev/null @@ -1,24 +0,0 @@ - - - WebRTC Performance Metrics - Video Engine - - - -

Video quality at different delays and packet loss rates (PSNR) Linux/Win/Mac

- - - -

Video quality at different delays and packet loss rates (SSIM) Linux/Win/Mac

- - - -

Time between rendered frames (ms) Linux/Win/Mac

- - - -

Time between rendered frames (ms) Linux/Win/Mac

- - - - - \ No newline at end of file diff --git a/tools/quality_tracking/dashboard/static/webrtc_logo.png b/tools/quality_tracking/dashboard/static/webrtc_logo.png deleted file mode 100644 index ce5ece077..000000000 Binary files a/tools/quality_tracking/dashboard/static/webrtc_logo.png and /dev/null differ diff --git a/tools/quality_tracking/dashboard/stylesheets/perf.css b/tools/quality_tracking/dashboard/stylesheets/perf.css deleted file mode 100644 index 7a98e7529..000000000 --- a/tools/quality_tracking/dashboard/stylesheets/perf.css +++ /dev/null @@ -1,21 +0,0 @@ -/******************************************************************** -* -* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. -* -* Use of this source code is governed by a BSD-style license -* that can be found in the LICENSE file in the root of the source -* tree. An additional intellectual property rights grant can be found -* in the file PATENTS. All contributing project authors may -* be found in the AUTHORS file in the root of the source tree. -* -*********************************************************************/ - -iframe { - width: 33%; /* We'll generally present 3 platforms. */ - border: 0; -} - -p { - font-family: verdana; - font-size: xx-large; -} diff --git a/tools/quality_tracking/dashboard/stylesheets/stylesheet.css b/tools/quality_tracking/dashboard/stylesheets/stylesheet.css deleted file mode 100644 index 992c57ff9..000000000 --- a/tools/quality_tracking/dashboard/stylesheets/stylesheet.css +++ /dev/null @@ -1,46 +0,0 @@ -/******************************************************************** -* -* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. -* -* Use of this source code is governed by a BSD-style license -* that can be found in the LICENSE file in the root of the source -* tree. An additional intellectual property rights grant can be found -* in the file PATENTS. All contributing project authors may -* be found in the AUTHORS file in the root of the source tree. -* -*********************************************************************/ - -.status_OK { - color: #FFFFFF; - background-color: #8fdf5f; -} - -.status_failed { - color: #FFFFFF; - background-color: #e98080; -} - -.status_building { - color: #666666; - background-color: #fffc6c; -} - -.status_warnings { - color: #000000; - background-color: #FFC343; -} - -.perf_links { - margin-bottom: 50px; - font-size: 200%; -} - -.status_cell { - width: 100px; - text-align: center; -} - -body { - margin-left: 35px; - margin-top: 25px; -} diff --git a/tools/quality_tracking/dashboard/templates/dashboard_template.html b/tools/quality_tracking/dashboard/templates/dashboard_template.html deleted file mode 100644 index a45bc17be..000000000 --- a/tools/quality_tracking/dashboard/templates/dashboard_template.html +++ /dev/null @@ -1,78 +0,0 @@ - - - - - WebRTC Quality Dashboard - - - - - - - - - -

Performance Metrics

-

- Chrome Video Quality | - Chrome Audio / Voice Engine Quality | - Chrome Audio Latency | - Video Engine -

-

Code Coverage History (Small / Medium Tests)

-
-

Code Coverage History (Large Tests)

-
- - diff --git a/tools/quality_tracking/dashboard/test/load_build_status_test.py b/tools/quality_tracking/dashboard/test/load_build_status_test.py deleted file mode 100755 index 7fc291038..000000000 --- a/tools/quality_tracking/dashboard/test/load_build_status_test.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python -#-*- coding: utf-8 -*- -# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. -# -# Use of this source code is governed by a BSD-style license -# that can be found in the LICENSE file in the root of the source -# tree. An additional intellectual property rights grant can be found -# in the file PATENTS. All contributing project authors may -# be found in the AUTHORS file in the root of the source tree. - -import unittest -from google.appengine.ext import testbed - -from add_build_status_data import BuildStatusData -import load_build_status - -class LoadBuildStatusTest(unittest.TestCase): - def setUp(self): - # First, create an instance of the Testbed class. - self.testbed = testbed.Testbed() - # Then activate the testbed, which prepares the service stubs for use. - self.testbed.activate() - # Next, declare which service stubs you want to use. - self.testbed.init_datastore_v3_stub() - - def test_returns_latest_nonbuilding_entries_when_loading_build_status(self): - BuildStatusData(bot_name="Bot1", revision=17, - build_number=499, status="OK").put() - BuildStatusData(bot_name="Bot2", revision=17, - build_number=505, status="OK").put() - BuildStatusData(bot_name="Bot3", revision=17, - build_number=344, status="failed").put() - BuildStatusData(bot_name="Bot1", revision=18, - build_number=499, status="building").put() - BuildStatusData(bot_name="Bot2", revision=18, - build_number=505, status="failed").put() - BuildStatusData(bot_name="Bot3", revision=18, - build_number=344, status="OK").put() - - loader = load_build_status.BuildStatusLoader() - result = loader.load_build_status_data() - - self.assertEqual(3, len(result)) - - # We make no guarantees on order, but we can use the fact that the testbed - # is deterministic to evaluate that the corrects bots were selected like so: - self.assertEqual("Bot1", result[0].bot_name) - self.assertEqual(17, result[0].revision) - self.assertEqual("OK", result[0].status) - - self.assertEqual("Bot3", result[1].bot_name) - self.assertEqual(18, result[1].revision) - self.assertEqual("OK", result[1].status) - - self.assertEqual("Bot2", result[2].bot_name) - self.assertEqual(18, result[2].revision) - self.assertEqual("failed", result[2].status) - - def test_returns_lkgr_for_single_green_revision(self): - BuildStatusData(bot_name="Bot1", revision=17, - build_number=499, status="OK").put() - BuildStatusData(bot_name="Bot2", revision=17, - build_number=505, status="OK").put() - BuildStatusData(bot_name="Bot3", revision=17, - build_number=344, status="OK").put() - - loader = load_build_status.BuildStatusLoader() - self.assertEqual(17, loader.compute_lkgr()) - - def test_returns_correct_lkgr_with_most_recent_revision_failed(self): - BuildStatusData(bot_name="Bot1", revision=17, - build_number=499, status="OK").put() - BuildStatusData(bot_name="Bot2", revision=17, - build_number=505, status="OK").put() - BuildStatusData(bot_name="Bot3", revision=17, - build_number=344, status="OK").put() - BuildStatusData(bot_name="Bot1", revision=18, - build_number=499, status="OK").put() - BuildStatusData(bot_name="Bot2", revision=18, - build_number=505, status="failed").put() - BuildStatusData(bot_name="Bot3", revision=18, - build_number=344, status="OK").put() - - loader = load_build_status.BuildStatusLoader() - self.assertEqual(17, loader.compute_lkgr()) - - def test_returns_none_if_no_revisions(self): - loader = load_build_status.BuildStatusLoader() - self.assertEqual(None, loader.compute_lkgr()) - - def test_returns_none_if_no_green_revisions(self): - BuildStatusData(bot_name="Bot2", revision=18, - build_number=505, status="failed").put() - - loader = load_build_status.BuildStatusLoader() - self.assertEqual(None, loader.compute_lkgr()) - - def test_skips_partially_building_revisions(self): - BuildStatusData(bot_name="Bot1", revision=18, - build_number=499, status="building").put() - BuildStatusData(bot_name="Bot2", revision=18, - build_number=505, status="OK").put() - BuildStatusData(bot_name="Bot1", revision=17, - build_number=344, status="OK").put() - - loader = load_build_status.BuildStatusLoader() - self.assertEqual(17, loader.compute_lkgr()) - - -if __name__ == '__main__': - unittest.main() diff --git a/tools/quality_tracking/dashboard_connection.py b/tools/quality_tracking/dashboard_connection.py deleted file mode 100644 index 632427be5..000000000 --- a/tools/quality_tracking/dashboard_connection.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python -#-*- coding: utf-8 -*- -# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. -# -# Use of this source code is governed by a BSD-style license -# that can be found in the LICENSE file in the root of the source -# tree. An additional intellectual property rights grant can be found -# in the file PATENTS. All contributing project authors may -# be found in the AUTHORS file in the root of the source tree. - -"""Contains utilities for communicating with the dashboard.""" - -import httplib -import shelve -import oauth.oauth as oauth - -import constants - - -class FailedToReadRequiredInputFile(Exception): - pass - - -class FailedToReportToDashboard(Exception): - pass - - -class DashboardConnection: - """Helper class for pushing data to the dashboard. - - This class deals with most of details for accessing protected resources - (i.e. data-writing operations) on the dashboard. Such operations are - authenticated using OAuth. This class requires a consumer secret and a - access token. - - The access token and consumer secrets are stored as files on disk in the - working directory of the scripts. Both files are created by the - request_oauth_permission script. - """ - - def __init__(self, consumer_key): - self.consumer_key_ = consumer_key - self.consumer_secret_ = None - self.access_token_string_ = None - - def read_required_files(self, consumer_secret_file, access_token_file): - """Reads required data for making OAuth requests. - - Args: - consumer_secret_file: A shelve file with an entry consumer_secret - containing the consumer secret in string form. - access_token_file: A shelve file with an entry access_token - containing the access token in string form. - """ - self.access_token_string_ = self._read_access_token(access_token_file) - self.consumer_secret_ = self._read_consumer_secret(consumer_secret_file) - - def send_post_request(self, url, parameters): - """Sends an OAuth request for a protected resource in the dashboard. - - Use this when you want to report new data to the dashboard. You must have - called the read_required_files method prior to calling this method, since - that method will read in the consumer secret and access token we need to - make the OAuth request. These concepts are described in the class - description. - - The server is expected to respond with HTTP status 200 and a completely - empty response if the call failed. The server may put diagnostic - information in the response. - - Args: - url: An absolute url within the dashboard domain, for example - http://webrtc-dashboard.appspot.com/add_coverage_data. - parameters: A dict which maps from POST parameter names to values. - - Raises: - FailedToReportToDashboard: If the dashboard didn't respond - with HTTP 200 to our request or if the response is non-empty. - """ - consumer = oauth.OAuthConsumer(self.consumer_key_, self.consumer_secret_) - access_token = oauth.OAuthToken.from_string(self.access_token_string_) - - oauth_request = oauth.OAuthRequest.from_consumer_and_token( - consumer, - token=access_token, - http_method='POST', - http_url=url, - parameters=parameters) - - signature_method_hmac_sha1 = oauth.OAuthSignatureMethod_HMAC_SHA1() - oauth_request.sign_request(signature_method_hmac_sha1, consumer, - access_token) - - connection = httplib.HTTPConnection(constants.DASHBOARD_SERVER) - - headers = {'Content-Type': 'application/x-www-form-urlencoded'} - connection.request('POST', url, body=oauth_request.to_postdata(), - headers=headers) - - response = connection.getresponse() - connection.close() - - if response.status != 200: - message = ('Failed to report to %s: got response %d (%s)' % - (url, response.status, response.reason)) - raise FailedToReportToDashboard(message) - - # The response content should be empty on success, so check that: - response_content = response.read() - if response_content: - message = ('Dashboard reported the following error: %s.' % - response_content) - raise FailedToReportToDashboard(message) - - def _read_access_token(self, filename): - return self._read_shelve(filename, 'access_token') - - def _read_consumer_secret(self, filename): - return self._read_shelve(filename, 'consumer_secret') - - @staticmethod - def _read_shelve(filename, key): - input_file = shelve.open(filename) - - if not input_file.has_key(key): - raise FailedToReadRequiredInputFile('Missing correct %s file in current ' - 'directory. You may have to run ' - 'request_oauth_permission.py.' % - filename) - - result = input_file[key] - input_file.close() - - return result diff --git a/tools/quality_tracking/oauth2 b/tools/quality_tracking/oauth2 deleted file mode 120000 index 63ab40baf..000000000 --- a/tools/quality_tracking/oauth2 +++ /dev/null @@ -1 +0,0 @@ -../third_party/oauth2/oauth2 \ No newline at end of file diff --git a/tools/quality_tracking/request_oauth_permission.py b/tools/quality_tracking/request_oauth_permission.py deleted file mode 100755 index 4e2fb37e7..000000000 --- a/tools/quality_tracking/request_oauth_permission.py +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env python -#-*- coding: utf-8 -*- -# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. -# -# Use of this source code is governed by a BSD-style license -# that can be found in the LICENSE file in the root of the source -# tree. An additional intellectual property rights grant can be found -# in the file PATENTS. All contributing project authors may -# be found in the AUTHORS file in the root of the source tree. - -"""This script request an access token from the appengine running the dashboard. - - The script is intended to be run manually whenever we wish to change which - dashboard administrator we act on behalf of when running the - track_coverage.py script. For example, this will be useful if the current - dashboard administrator leaves the project. This script can also be used to - launch a new dashboard if that is desired. - - This script should be run on the build bot which runs the track_coverage.py - script. This script will present a link during its execution, which the new - administrator should follow and then click approve on the web page that - appears. The new administrator should have admin rights on the coverage - dashboard, otherwise the track_* scripts will not work. - - If successful, this script will write the access token to a file access.token - in the current directory, which later can be read by the track_* scripts. - The token is stored in string form (as reported by the web server) using the - shelve module. The consumer secret passed in as an argument to this script - will also similarly be stored in a file consumer.secret. The shelve keys - will be 'access_token' and 'consumer_secret', respectively. -""" - -import shelve -import sys -import urlparse -import oauth2 as oauth - -import constants - - -class FailedToRequestPermissionException(Exception): - pass - - -def _ensure_token_response_is_200(response, queried_url, token_type): - if response.status != 200: - raise FailedToRequestPermissionException('Failed to request %s from %s: ' - 'received status %d, reason %s.' % - (token_type, - queried_url, - response.status, - response.reason)) - - -def _request_unauthorized_token(consumer, request_token_url): - """Requests the initial token from the dashboard service. - - Given that the response from the server is correct, we will return a - dictionary containing oauth_token and oauth_token_secret mapped to the - token and secret value, respectively. - """ - client = oauth.Client(consumer) - - try: - response, content = client.request(request_token_url, 'POST') - except AttributeError as error: - # This catch handler is here since we'll get very confusing messages - # if the target server is down for some reason. - raise FailedToRequestPermissionException('Failed to request token: ' - 'the dashboard is likely down.', - error) - - _ensure_token_response_is_200(response, request_token_url, - 'unauthorized token') - - return dict(urlparse.parse_qsl(content)) - - -def _ask_user_to_authorize_us(unauthorized_token): - """This function will block until the user enters y + newline.""" - print 'Go to the following link in your browser:' - print '%s?oauth_token=%s' % (constants.AUTHORIZE_TOKEN_URL, - unauthorized_token['oauth_token']) - - accepted = 'n' - while accepted.lower() != 'y': - accepted = raw_input('Have you authorized me yet? (y/n) ') - - -def _request_access_token(consumer, unauthorized_token): - token = oauth.Token(unauthorized_token['oauth_token'], - unauthorized_token['oauth_token_secret']) - client = oauth.Client(consumer, token) - response, content = client.request(constants.ACCESS_TOKEN_URL, 'POST') - - _ensure_token_response_is_200(response, constants.ACCESS_TOKEN_URL, - 'access token') - - return content - - -def _write_access_token_to_file(access_token, filename): - output = shelve.open(filename) - output['access_token'] = access_token - output.close() - - print 'Wrote the access token to the file %s.' % filename - - -def _write_consumer_secret_to_file(consumer_secret, filename): - output = shelve.open(filename) - output['consumer_secret'] = consumer_secret - output.close() - - print 'Wrote the consumer secret to the file %s.' % filename - - -def _main(): - if len(sys.argv) != 2: - print ('Usage: %s .\n\nThe consumer secret is an OAuth ' - 'concept and is obtained from the Google Accounts domain dashboard.' - % sys.argv[0]) - return - - consumer_secret = sys.argv[1] - consumer = oauth.Consumer(constants.CONSUMER_KEY, consumer_secret) - - unauthorized_token = _request_unauthorized_token(consumer, - constants.REQUEST_TOKEN_URL) - - _ask_user_to_authorize_us(unauthorized_token) - - access_token_string = _request_access_token(consumer, unauthorized_token) - - _write_access_token_to_file(access_token_string, constants.ACCESS_TOKEN_FILE) - _write_consumer_secret_to_file(consumer_secret, - constants.CONSUMER_SECRET_FILE) - -if __name__ == '__main__': - _main() diff --git a/tools/quality_tracking/track_coverage.py b/tools/quality_tracking/track_coverage.py deleted file mode 100755 index 8c063c144..000000000 --- a/tools/quality_tracking/track_coverage.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python -#-*- coding: utf-8 -*- -# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. -# -# Use of this source code is governed by a BSD-style license -# that can be found in the LICENSE file in the root of the source -# tree. An additional intellectual property rights grant can be found -# in the file PATENTS. All contributing project authors may -# be found in the AUTHORS file in the root of the source tree. - -"""This script grabs and reports coverage information. - - It grabs coverage information from the latest Linux 32-bit build and - pushes it to the coverage tracker, enabling us to track code coverage - over time. This script is intended to run on the 32-bit Linux slave. - - This script requires an access.token file in the current directory, as - generated by the request_oauth_permission.py script. It also expects a file - customer.secret with a single line containing the customer secret. The - customer secret is an OAuth concept and is received when one registers the - application with the App Engine running the dashboard. - - The script assumes that all coverage data is stored under - /home//www. -""" - -import os -import re -import sys - -import constants -import dashboard_connection - - -class FailedToParseCoverageHtml(Exception): - pass - - -class CouldNotFindCoverageDirectory(Exception): - pass - - -def _find_latest_build_coverage(www_directory_contents, coverage_www_dir, - directory_prefix): - """Finds the most recent coverage directory in the directory listing. - - We assume here that build numbers keep rising and never wrap around. - - Args: - www_directory_contents: A list of entries in the coverage directory. - coverage_www_dir: The coverage directory on the bot. - directory_prefix: Coverage directories have the form , - and the prefix is different on different bots. The prefix is - generally the builder name, such as Linux32DBG. - - Returns: - The most recent directory name. - - Raises: - CouldNotFindCoverageDirectory: if we failed to find coverage data. - """ - - found_build_numbers = [] - for entry in www_directory_contents: - match = re.match(directory_prefix + '(\d+)', entry) - if match is not None: - found_build_numbers.append(int(match.group(1))) - - if not found_build_numbers: - raise CouldNotFindCoverageDirectory('Error: Found no directories %s* ' - 'in directory %s.' % - (directory_prefix, coverage_www_dir)) - - most_recent = max(found_build_numbers) - return directory_prefix + str(most_recent) - - -def _grab_coverage_percentage(label, index_html_contents): - """Extracts coverage from a LCOV coverage report. - - Grabs coverage by assuming that the label in the coverage HTML report - is close to the actual number and that the number is followed by a space - and a percentage sign. - """ - match = re.search(']*>' + label + '.*?(\d+\.\d) %', - index_html_contents, re.DOTALL) - if match is None: - raise FailedToParseCoverageHtml('Missing coverage at label "%s".' % label) - - try: - return float(match.group(1)) - except ValueError: - raise FailedToParseCoverageHtml('%s is not a float.' % match.group(1)) - - -def _report_coverage_to_dashboard(dashboard, line_coverage, function_coverage, - branch_coverage, report_category): - parameters = {'line_coverage': '%f' % line_coverage, - 'function_coverage': '%f' % function_coverage, - 'branch_coverage': '%f' % branch_coverage, - 'report_category': report_category, - } - - dashboard.send_post_request(constants.ADD_COVERAGE_DATA_URL, parameters) - - -def _main(report_category, directory_prefix): - """Grabs coverage data from disk on a bot and publishes it. - - Args: - report_category: The kind of coverage to report. The dashboard - application decides what is acceptable here (see - dashboard/add_coverage_data.py for more information). - directory_prefix: This bot's coverage directory prefix. Generally a bot's - coverage directories will have the form , - like Linux32DBG_345. - """ - dashboard = dashboard_connection.DashboardConnection(constants.CONSUMER_KEY) - dashboard.read_required_files(constants.CONSUMER_SECRET_FILE, - constants.ACCESS_TOKEN_FILE) - - coverage_www_dir = constants.BUILD_BOT_COVERAGE_WWW_DIRECTORY - www_dir_contents = os.listdir(coverage_www_dir) - latest_build_directory = _find_latest_build_coverage(www_dir_contents, - coverage_www_dir, - directory_prefix) - - index_html_path = os.path.join(coverage_www_dir, latest_build_directory, - 'index.html') - index_html_file = open(index_html_path) - whole_file = index_html_file.read() - - line_coverage = _grab_coverage_percentage('Lines:', whole_file) - function_coverage = _grab_coverage_percentage('Functions:', whole_file) - branch_coverage = _grab_coverage_percentage('Branches:', whole_file) - - _report_coverage_to_dashboard(dashboard, line_coverage, function_coverage, - branch_coverage, report_category) - - -def _parse_args(): - if len(sys.argv) != 3: - print ('Usage: %s \n\n' - 'The coverage category describes the kind of coverage you are ' - 'uploading. Known acceptable values are small_medium_tests and' - 'large_tests. The directory prefix is what the directories in %s ' - 'are prefixed on this bot (such as Linux32DBG_).' % - (sys.argv[0], constants.BUILD_BOT_COVERAGE_WWW_DIRECTORY)) - return (None, None) - return (sys.argv[1], sys.argv[2]) - - -if __name__ == '__main__': - category, dir_prefix = _parse_args() - if category: - _main(category, dir_prefix) -