diff --git a/.gitignore b/.gitignore index 23651e65c..7fc252559 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,7 @@ /third_party/libvpx /third_party/libyuv /third_party/llvm-build +/third_party/oauth2 /third_party/protobuf /third_party/valgrind /third_party/yasm diff --git a/DEPS b/DEPS index 6b7c07ef1..31fdfd0d9 100644 --- a/DEPS +++ b/DEPS @@ -73,6 +73,7 @@ deps = { "trunk/third_party/yasm/source/patched-yasm": Var("chromium_trunk") + "/deps/third_party/yasm/patched-yasm@73761", + # Used by libjpeg-turbo "trunk/third_party/yasm/binaries": Var("chromium_trunk") + "/deps/third_party/yasm/binaries@74228", @@ -82,9 +83,14 @@ deps = { "trunk/third_party/libyuv": (Var("googlecode_url") % "libyuv") + "/trunk@121", - + + # Used by tools/coverage/dashboard and tools/python_charts "trunk/third_party/google-visualization-python": (Var("googlecode_url") % "google-visualization-python") + "/trunk@15", + + # Used by tools/coverage + "trunk/third_party/oauth2": + "https://github.com/simplegeo/python-oauth2.git@a83f4a297336b631e75cba102910c19231518159" } deps_os = { diff --git a/tools/coverage/README b/tools/coverage/README index 12843fbe6..faf3e7a85 100644 --- a/tools/coverage/README +++ b/tools/coverage/README @@ -7,10 +7,18 @@ the track_coverage.py script is intended to run on the build bot as a cron job and extract the data from there. The dashboard doesn't care how often this script runs, but running each hour should be more than enough. -The track_coverage.py script communicates with the dashboard using plain GET -requests (that, and POST, are basically the only way to get data into a -appengine application such as the dashboard). The dashboard is intented to -run on the Google appengine. +The track_coverage.py script uses OAuth to authenticate itself. In order to do +this, it needs two files: consumer.secret and access.token. The consumer secret +is known within the organization and is stored in a plain file on the bot +running the scripts (we don't want to check in this secret in the code in the +public repository). The consumer secret is a plain file with a single line +containing the secret string. + +The access.token file is generated by request_oauth_permission.py. It does this +by going through the three-legged OAuth authorization process. An administrator +of the dashboard must approve the request from the script. Once that is done, +access.token will be written and track_coverage.py will be able to report +results. HOW TO RUN LOCALLY: Follow the following instructions: diff --git a/tools/coverage/dashboard/dashboard.py b/tools/coverage/dashboard/dashboard.py index 328343863..3d82e239c 100644 --- a/tools/coverage/dashboard/dashboard.py +++ b/tools/coverage/dashboard/dashboard.py @@ -12,11 +12,18 @@ __author__ = 'phoglund@webrtc.org (Patrik Hoglund)' import datetime +from google.appengine.api import oauth +from google.appengine.api import users from google.appengine.ext import db import webapp2 import gviz_api +class UserNotAuthenticatedException(Exception): + """Gets thrown if a user is not permitted to store coverage data.""" + pass + + class CoverageData(db.Model): """This represents one coverage report from the build bot.""" date = db.DateTimeProperty(required=True) @@ -40,8 +47,8 @@ class ShowDashboard(webapp2.RequestHandler): page_template = template_file.read() template_file.close() except IOError as exception: - self.ShowErrorPage('Cannot open page template file: %s
Details: %s' % - (page_template_filename, exception)) + self._show_error_page('Cannot open page template file: %s
Details: %s' + % (page_template_filename, exception)) return coverage_entries = db.GqlQuery('SELECT * ' @@ -65,18 +72,24 @@ class ShowDashboard(webapp2.RequestHandler): # Fill in the template with the data and respond: self.response.write(page_template % vars()) - def ShowErrorPage(self, error_message): + def _show_error_page(self, error_message): self.response.write('%s' % error_message) class AddCoverageData(webapp2.RequestHandler): """Used to report coverage data. - It will verify the data, but not the sender. Thus, it should be secured - more properly if accessible from an outside network. + The user is required to have obtained an OAuth access token from an + administrator for this application earlier. """ - def get(self): + def post(self): + try: + self._authenticate_user() + except UserNotAuthenticatedException as exception: + self._show_error_page('Failed to authenticate user: %s' % exception) + return + try: posix_time = int(self.request.get('date')) parsed_date = datetime.datetime.fromtimestamp(posix_time) @@ -84,8 +97,8 @@ class AddCoverageData(webapp2.RequestHandler): line_coverage = float(self.request.get('line_coverage')) function_coverage = float(self.request.get('function_coverage')) except ValueError as exception: - self.ShowErrorPage('Invalid parameter in request. Details: %s' % - exception) + self._show_error_page('Invalid parameter in request. Details: %s' % + exception) return item = CoverageData(date=parsed_date, @@ -93,7 +106,22 @@ class AddCoverageData(webapp2.RequestHandler): function_coverage=function_coverage) item.put() - def ShowErrorPage(self, error_message): + def _authenticate_user(self): + try: + if oauth.is_current_user_admin(): + # The user on whose behalf we are acting is indeed an administrator + # of this application, so we're good to go. + return + else: + raise UserNotAuthenticatedException('We are acting on behalf of ' + 'user %s, but that user is not ' + 'an administrator.' % + oauth.get_current_user()) + except oauth.OAuthRequestError as exception: + raise UserNotAuthenticatedException('Invalid OAuth request: %s' % + exception) + + def _show_error_page(self, error_message): self.response.write('%s' % error_message) app = webapp2.WSGIApplication([('/', ShowDashboard), diff --git a/tools/coverage/oauth2 b/tools/coverage/oauth2 new file mode 120000 index 000000000..a4d1dde74 --- /dev/null +++ b/tools/coverage/oauth2 @@ -0,0 +1 @@ +../../third_party/oauth2/oauth2/ \ No newline at end of file diff --git a/tools/coverage/request_oauth_permission.py b/tools/coverage/request_oauth_permission.py new file mode 100755 index 000000000..bad0902bd --- /dev/null +++ b/tools/coverage/request_oauth_permission.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python +#-*- coding: utf-8 -*- +# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""This script request an access token from the appengine running the dashboard. + + The script is intended to be run manually whenever we wish to change which + dashboard administrator we act on behalf of when running the + track_coverage.py script. For example, this will be useful if the current + dashboard administrator leaves the project. + + This script should be run on the build bot which runs the track_coverage.py + script. This script will present a link during its execution, which the new + administrator should follow and then click approve on the web page that + appears. The new administrator should have admin rights on the coverage + dashboard, otherwise the track_coverage.py will not work. + + If successful, this script will write the access token to a file access.token + in the current directory, which later can be read by track_coverage.py. + The token is stored in string form (as reported by the web server) using the + shelve module. +""" + +__author__ = 'phoglund@webrtc.org (Patrik Höglund)' + +import shelve +import sys +import urlparse +import oauth2 as oauth + + +class FailedToRequestPermissionException(Exception): + pass + + +# This identifies our application using the information we got when we +# registered the application on Google appengine. +# TODO(phoglund): update to the right value when we have registered the app. +DASHBOARD_SERVER = 'http://127.0.0.1:8080' +CONSUMER_KEY = DASHBOARD_SERVER + +REQUEST_TOKEN_URL = DASHBOARD_SERVER + '/_ah/OAuthGetRequestToken' +AUTHORIZE_TOKEN_URL = DASHBOARD_SERVER + '/_ah/OAuthAuthorizeToken' +ACCESS_TOKEN_URL = DASHBOARD_SERVER + '/_ah/OAuthGetAccessToken' + + +def _ensure_token_response_is_200(response, queried_url, token_type): + if response.status != 200: + raise FailedToRequestPermissionException('Failed to request %s from %s: ' + 'received status %d, reason %s.' % + (token_type, + queried_url, + response.status, + response.reason)) + +def _request_unauthorized_token(consumer, request_token_url): + """Requests the initial token from the dashboard service. + + Given that the response from the server is correct, we will return a + dictionary containing oauth_token and oauth_token_secret mapped to the + token and secret value, respectively. + """ + client = oauth.Client(consumer) + + try: + response, content = client.request(request_token_url, 'POST') + except AttributeError as error: + # This catch handler is here since we'll get very confusing messages + # if the target server is down for some reason. + raise FailedToRequestPermissionException("Failed to request token: " + "the dashboard is likely down.", + error) + + _ensure_token_response_is_200(response, request_token_url, + "unauthorized token") + + return dict(urlparse.parse_qsl(content)) + + +def _ask_user_to_authorize_us(unauthorized_token): + """This function will block until the user enters y + newline.""" + print 'Go to the following link in your browser:' + print '%s?oauth_token=%s' % (AUTHORIZE_TOKEN_URL, + unauthorized_token['oauth_token']) + + accepted = 'n' + while accepted.lower() != 'y': + accepted = raw_input('Have you authorized me yet? (y/n) ') + + +def _request_access_token(consumer, unauthorized_token): + token = oauth.Token(unauthorized_token['oauth_token'], + unauthorized_token['oauth_token_secret']) + client = oauth.Client(consumer, token) + response, content = client.request(ACCESS_TOKEN_URL, 'POST') + + _ensure_token_response_is_200(response, ACCESS_TOKEN_URL, "access token") + + return content + + +def _write_access_token_to_file(access_token, filename): + output = shelve.open(filename) + output['access_token'] = access_token + output.close() + + print 'Wrote the access token to the file %s.' % filename + + +def _main(): + if len(sys.argv) != 2: + print ('Usage: %s .\n\nThe consumer secret is an OAuth ' + 'concept and is obtained from the appengine running the dashboard.' % + sys.argv[0]) + return + + consumer_secret = sys.argv[1] + consumer = oauth.Consumer(CONSUMER_KEY, consumer_secret) + + unauthorized_token = _request_unauthorized_token(consumer, REQUEST_TOKEN_URL) + + _ask_user_to_authorize_us(unauthorized_token) + + access_token_string = _request_access_token(consumer, unauthorized_token) + + _write_access_token_to_file(access_token_string, 'access.token') + +if __name__ == '__main__': + _main() diff --git a/tools/coverage/track_coverage.py b/tools/coverage/track_coverage.py index 34cf12562..8bc9f3767 100755 --- a/tools/coverage/track_coverage.py +++ b/tools/coverage/track_coverage.py @@ -13,6 +13,15 @@ It grabs coverage information from the latest Linux 32-bit build and pushes it to the coverage tracker, enabling us to track code coverage over time. This script is intended to run on the 32-bit Linux slave. + + This script requires an access.token file in the current directory, as + generated by the request_oauth_permission.py script. It also expects a file + customer.secret with a single line containing the customer secret. The + customer secret is an OAuth concept and is received when one registers the + application with the appengine running the dashboard. + + The script assumes that all coverage data is stored under + /home//www. """ __author__ = 'phoglund@webrtc.org (Patrik Höglund)' @@ -20,8 +29,10 @@ __author__ = 'phoglund@webrtc.org (Patrik Höglund)' import httplib import os import re +import shelve import sys import time +import oauth.oauth as oauth # The build-bot user which runs build bot jobs. BUILD_BOT_USER = 'phoglund' @@ -29,6 +40,8 @@ BUILD_BOT_USER = 'phoglund' # The server to send coverage data to. # TODO(phoglund): replace with real server once we get it up. DASHBOARD_SERVER = 'localhost:8080' +CONSUMER_KEY = DASHBOARD_SERVER +ADD_COVERAGE_DATA_URL = '/add_coverage_data' class FailedToParseCoverageHtml(Exception): @@ -39,6 +52,35 @@ class FailedToReportToDashboard(Exception): pass +class FailedToReadRequiredInputFile(Exception): + pass + + +def _read_access_token_from_file(filename): + input_file = shelve.open(filename) + + if not input_file.has_key('access_token'): + raise FailedToReadRequiredInputFile('Missing %s file in current directory. ' + 'You may have to run ' + 'request_oauth_permission.py.') + + return oauth.OAuthToken.from_string(input_file['access_token']) + + +def _read_customer_secret_from_file(filename): + try: + input_file = open(filename, 'r') + except IOError as error: + raise FailedToReadRequiredInputFile(error) + + whole_file = input_file.read() + if '\n' in whole_file or not whole_file.strip(): + raise FailedToReadRequiredInputFile('Expected a single line with the ' + 'customer secret in file %s.' % + filename) + return whole_file + + def _find_latest_32bit_debug_build(www_directory_contents): # Build directories have the form Linux32bitDebug_. There may be other # directories in the list though, for instance for other build configurations. @@ -73,13 +115,30 @@ def _grab_coverage_percentage(label, index_html_contents): raise FailedToParseCoverageHtml('%s is not a float.' % match.group(1)) -def _report_coverage_to_dashboard(now, line_coverage, function_coverage): - request_string = ('/add_coverage_data?' - 'date=%d&line_coverage=%f&function_coverage=%f' % - (now, line_coverage, function_coverage)) +def _report_coverage_to_dashboard(now, line_coverage, function_coverage, + access_token, consumer_key, consumer_secret): + parameters = {'date': '%d' % now, + 'line_coverage': '%f' % line_coverage, + 'function_coverage': '%f' % function_coverage + } + consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) + create_oauth_request = oauth.OAuthRequest.from_consumer_and_token + oauth_request = create_oauth_request(consumer, + token=access_token, + http_method='POST', + http_url=ADD_COVERAGE_DATA_URL, + parameters=parameters) + + signature_method_hmac_sha1 = oauth.OAuthSignatureMethod_HMAC_SHA1() + oauth_request.sign_request(signature_method_hmac_sha1, consumer, access_token) connection = httplib.HTTPConnection(DASHBOARD_SERVER) - connection.request('GET', request_string) + + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + connection.request('POST', ADD_COVERAGE_DATA_URL, + body=oauth_request.to_postdata(), + headers=headers) + response = connection.getresponse() if response.status != 200: message = ('Error: Failed to report to %s%s: got response %d (%s)' % @@ -96,6 +155,9 @@ def _report_coverage_to_dashboard(now, line_coverage, function_coverage): def _main(): + access_token = _read_access_token_from_file('access.token') + customer_secret = _read_customer_secret_from_file('customer.secret') + coverage_www_dir = os.path.join('/home', BUILD_BOT_USER, 'www') www_dir_contents = os.listdir(coverage_www_dir) @@ -114,7 +176,8 @@ def _main(): function_coverage = _grab_coverage_percentage('Functions:', whole_file) now = int(time.time()) - _report_coverage_to_dashboard(now, line_coverage, function_coverage) + _report_coverage_to_dashboard(now, line_coverage, function_coverage, + access_token, CONSUMER_KEY, customer_secret) if __name__ == '__main__': _main()