generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Completed implementation of oauth in coverage scripts.

Browse Source 
The access token is now transferred between the request and track scripts.

Almost completed the implementation, using oauth2.

Initial, incomplete implementation of the permission-request script.

The coverage tracking is broken temporarily, but it can now make OAuth requests.

BUG=
TEST=

Review URL: https://webrtc-codereview.appspot.com/366002

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1530 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
phoglund@webrtc.org 2012-01-24 14:44:51 +00:00
parent 28a5cb29ab
commit 96c39d1f73
7 changed files with 262 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -49,6 +49,7 @@
/third_party/libvpx
/third_party/libyuv
/third_party/llvm-build
/third_party/oauth2
/third_party/protobuf
/third_party/valgrind
/third_party/yasm

8
DEPS
View File

@ -73,6 +73,7 @@ deps = {
"trunk/third_party/yasm/source/patched-yasm":
Var("chromium_trunk") + "/deps/third_party/yasm/patched-yasm@73761",
# Used by libjpeg-turbo
"trunk/third_party/yasm/binaries":
Var("chromium_trunk") + "/deps/third_party/yasm/binaries@74228",
@ -82,9 +83,14 @@ deps = {
"trunk/third_party/libyuv":
(Var("googlecode_url") % "libyuv") + "/trunk@121",
# Used by tools/coverage/dashboard and tools/python_charts
"trunk/third_party/google-visualization-python":
(Var("googlecode_url") % "google-visualization-python") + "/trunk@15",
# Used by tools/coverage
"trunk/third_party/oauth2":
"https://github.com/simplegeo/python-oauth2.git@a83f4a297336b631e75cba102910c19231518159"
}
deps_os = {

16
tools/coverage/README
View File

@ -7,10 +7,18 @@ the track_coverage.py script is intended to run on the build bot as a cron job
and extract the data from there. The dashboard doesn't care how often this
script runs, but running each hour should be more than enough.
The track_coverage.py script communicates with the dashboard using plain GET
requests (that, and POST, are basically the only way to get data into a
appengine application such as the dashboard). The dashboard is intented to
run on the Google appengine.
The track_coverage.py script uses OAuth to authenticate itself. In order to do
this, it needs two files: consumer.secret and access.token. The consumer secret
is known within the organization and is stored in a plain file on the bot
running the scripts (we don't want to check in this secret in the code in the
public repository). The consumer secret is a plain file with a single line
containing the secret string.
The access.token file is generated by request_oauth_permission.py. It does this
by going through the three-legged OAuth authorization process. An administrator
of the dashboard must approve the request from the script. Once that is done,
access.token will be written and track_coverage.py will be able to report
results.
HOW TO RUN LOCALLY:
Follow the following instructions:

46
tools/coverage/dashboard/dashboard.py
View File

@ -12,11 +12,18 @@
__author__ = 'phoglund@webrtc.org (Patrik Hoglund)'
import datetime
from google.appengine.api import oauth
from google.appengine.api import users
from google.appengine.ext import db
import webapp2
import gviz_api
class UserNotAuthenticatedException(Exception):
"""Gets thrown if a user is not permitted to store coverage data."""
pass
class CoverageData(db.Model):
"""This represents one coverage report from the build bot."""
date = db.DateTimeProperty(required=True)
@ -40,8 +47,8 @@ class ShowDashboard(webapp2.RequestHandler):
page_template = template_file.read()
template_file.close()
except IOError as exception:
self.ShowErrorPage('Cannot open page template file: %s<br>Details: %s' %
(page_template_filename, exception))
self._show_error_page('Cannot open page template file: %s<br>Details: %s'
% (page_template_filename, exception))
return
coverage_entries = db.GqlQuery('SELECT * '
@ -65,18 +72,24 @@ class ShowDashboard(webapp2.RequestHandler):
# Fill in the template with the data and respond:
self.response.write(page_template % vars())
def ShowErrorPage(self, error_message):
def _show_error_page(self, error_message):
self.response.write('<html><body>%s</body></html>' % error_message)
class AddCoverageData(webapp2.RequestHandler):
"""Used to report coverage data.
It will verify the data, but not the sender. Thus, it should be secured
more properly if accessible from an outside network.
The user is required to have obtained an OAuth access token from an
administrator for this application earlier.
"""
def get(self):
def post(self):
try:
self._authenticate_user()
except UserNotAuthenticatedException as exception:
self._show_error_page('Failed to authenticate user: %s' % exception)
return
try:
posix_time = int(self.request.get('date'))
parsed_date = datetime.datetime.fromtimestamp(posix_time)
@ -84,8 +97,8 @@ class AddCoverageData(webapp2.RequestHandler):
line_coverage = float(self.request.get('line_coverage'))
function_coverage = float(self.request.get('function_coverage'))
except ValueError as exception:
self.ShowErrorPage('Invalid parameter in request. Details: %s' %
exception)
self._show_error_page('Invalid parameter in request. Details: %s' %
exception)
return
item = CoverageData(date=parsed_date,
@ -93,7 +106,22 @@ class AddCoverageData(webapp2.RequestHandler):
function_coverage=function_coverage)
item.put()
def ShowErrorPage(self, error_message):
def _authenticate_user(self):
try:
if oauth.is_current_user_admin():
# The user on whose behalf we are acting is indeed an administrator
# of this application, so we're good to go.
return
else:
raise UserNotAuthenticatedException('We are acting on behalf of '
'user %s, but that user is not '
'an administrator.' %
oauth.get_current_user())
except oauth.OAuthRequestError as exception:
raise UserNotAuthenticatedException('Invalid OAuth request: %s' %
exception)
def _show_error_page(self, error_message):
self.response.write('<html><body>%s</body></html>' % error_message)
app = webapp2.WSGIApplication([('/', ShowDashboard),

1
tools/coverage/oauth2 Symbolic link
View File

@ -0,0 +1 @@
../../third_party/oauth2/oauth2/

135
tools/coverage/request_oauth_permission.py Executable file
View File

@ -0,0 +1,135 @@
#!/usr/bin/env python
#-*- coding: utf-8 -*-
# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
"""This script request an access token from the appengine running the dashboard.
The script is intended to be run manually whenever we wish to change which
dashboard administrator we act on behalf of when running the
track_coverage.py script. For example, this will be useful if the current
dashboard administrator leaves the project.
This script should be run on the build bot which runs the track_coverage.py
script. This script will present a link during its execution, which the new
administrator should follow and then click approve on the web page that
appears. The new administrator should have admin rights on the coverage
dashboard, otherwise the track_coverage.py will not work.
If successful, this script will write the access token to a file access.token
in the current directory, which later can be read by track_coverage.py.
The token is stored in string form (as reported by the web server) using the
shelve module.
"""
__author__ = 'phoglund@webrtc.org (Patrik Höglund)'
import shelve
import sys
import urlparse
import oauth2 as oauth
class FailedToRequestPermissionException(Exception):
pass
# This identifies our application using the information we got when we
# registered the application on Google appengine.
# TODO(phoglund): update to the right value when we have registered the app.
DASHBOARD_SERVER = 'http://127.0.0.1:8080'
CONSUMER_KEY = DASHBOARD_SERVER
REQUEST_TOKEN_URL = DASHBOARD_SERVER + '/_ah/OAuthGetRequestToken'
AUTHORIZE_TOKEN_URL = DASHBOARD_SERVER + '/_ah/OAuthAuthorizeToken'
ACCESS_TOKEN_URL = DASHBOARD_SERVER + '/_ah/OAuthGetAccessToken'
def _ensure_token_response_is_200(response, queried_url, token_type):
if response.status != 200:
raise FailedToRequestPermissionException('Failed to request %s from %s: '
'received status %d, reason %s.' %
(token_type,
queried_url,
response.status,
response.reason))
def _request_unauthorized_token(consumer, request_token_url):
"""Requests the initial token from the dashboard service.
Given that the response from the server is correct, we will return a
dictionary containing oauth_token and oauth_token_secret mapped to the
token and secret value, respectively.
"""
client = oauth.Client(consumer)
try:
response, content = client.request(request_token_url, 'POST')
except AttributeError as error:
# This catch handler is here since we'll get very confusing messages
# if the target server is down for some reason.
raise FailedToRequestPermissionException("Failed to request token: "
"the dashboard is likely down.",
error)
_ensure_token_response_is_200(response, request_token_url,
"unauthorized token")
return dict(urlparse.parse_qsl(content))
def _ask_user_to_authorize_us(unauthorized_token):
"""This function will block until the user enters y + newline."""
print 'Go to the following link in your browser:'
print '%s?oauth_token=%s' % (AUTHORIZE_TOKEN_URL,
unauthorized_token['oauth_token'])
accepted = 'n'
while accepted.lower() != 'y':
accepted = raw_input('Have you authorized me yet? (y/n) ')
def _request_access_token(consumer, unauthorized_token):
token = oauth.Token(unauthorized_token['oauth_token'],
unauthorized_token['oauth_token_secret'])
client = oauth.Client(consumer, token)
response, content = client.request(ACCESS_TOKEN_URL, 'POST')
_ensure_token_response_is_200(response, ACCESS_TOKEN_URL, "access token")
return content
def _write_access_token_to_file(access_token, filename):
output = shelve.open(filename)
output['access_token'] = access_token
output.close()
print 'Wrote the access token to the file %s.' % filename
def _main():
if len(sys.argv) != 2:
print ('Usage: %s <consumer secret>.\n\nThe consumer secret is an OAuth '
'concept and is obtained from the appengine running the dashboard.' %
sys.argv[0])
return
consumer_secret = sys.argv[1]
consumer = oauth.Consumer(CONSUMER_KEY, consumer_secret)
unauthorized_token = _request_unauthorized_token(consumer, REQUEST_TOKEN_URL)
_ask_user_to_authorize_us(unauthorized_token)
access_token_string = _request_access_token(consumer, unauthorized_token)
_write_access_token_to_file(access_token_string, 'access.token')
if __name__ == '__main__':
_main()

75
tools/coverage/track_coverage.py
View File

@ -13,6 +13,15 @@
It grabs coverage information from the latest Linux 32-bit build and
pushes it to the coverage tracker, enabling us to track code coverage
over time. This script is intended to run on the 32-bit Linux slave.
This script requires an access.token file in the current directory, as
generated by the request_oauth_permission.py script. It also expects a file
customer.secret with a single line containing the customer secret. The
customer secret is an OAuth concept and is received when one registers the
application with the appengine running the dashboard.
The script assumes that all coverage data is stored under
/home/<build bot user>/www.
"""
__author__ = 'phoglund@webrtc.org (Patrik Höglund)'
@ -20,8 +29,10 @@ __author__ = 'phoglund@webrtc.org (Patrik Höglund)'
import httplib
import os
import re
import shelve
import sys
import time
import oauth.oauth as oauth
# The build-bot user which runs build bot jobs.
BUILD_BOT_USER = 'phoglund'
@ -29,6 +40,8 @@ BUILD_BOT_USER = 'phoglund'
# The server to send coverage data to.
# TODO(phoglund): replace with real server once we get it up.
DASHBOARD_SERVER = 'localhost:8080'
CONSUMER_KEY = DASHBOARD_SERVER
ADD_COVERAGE_DATA_URL = '/add_coverage_data'
class FailedToParseCoverageHtml(Exception):
@ -39,6 +52,35 @@ class FailedToReportToDashboard(Exception):
pass
class FailedToReadRequiredInputFile(Exception):
pass
def _read_access_token_from_file(filename):
input_file = shelve.open(filename)
if not input_file.has_key('access_token'):
raise FailedToReadRequiredInputFile('Missing %s file in current directory. '
'You may have to run '
'request_oauth_permission.py.')
return oauth.OAuthToken.from_string(input_file['access_token'])
def _read_customer_secret_from_file(filename):
try:
input_file = open(filename, 'r')
except IOError as error:
raise FailedToReadRequiredInputFile(error)
whole_file = input_file.read()
if '\n' in whole_file or not whole_file.strip():
raise FailedToReadRequiredInputFile('Expected a single line with the '
'customer secret in file %s.' %
filename)
return whole_file
def _find_latest_32bit_debug_build(www_directory_contents):
# Build directories have the form Linux32bitDebug_<number>. There may be other
# directories in the list though, for instance for other build configurations.
@ -73,13 +115,30 @@ def _grab_coverage_percentage(label, index_html_contents):
raise FailedToParseCoverageHtml('%s is not a float.' % match.group(1))
def _report_coverage_to_dashboard(now, line_coverage, function_coverage):
request_string = ('/add_coverage_data?'
'date=%d&line_coverage=%f&function_coverage=%f' %
(now, line_coverage, function_coverage))
def _report_coverage_to_dashboard(now, line_coverage, function_coverage,
access_token, consumer_key, consumer_secret):
parameters = {'date': '%d' % now,
'line_coverage': '%f' % line_coverage,
'function_coverage': '%f' % function_coverage
}
consumer = oauth.OAuthConsumer(consumer_key, consumer_secret)
create_oauth_request = oauth.OAuthRequest.from_consumer_and_token
oauth_request = create_oauth_request(consumer,
token=access_token,
http_method='POST',
http_url=ADD_COVERAGE_DATA_URL,
parameters=parameters)
signature_method_hmac_sha1 = oauth.OAuthSignatureMethod_HMAC_SHA1()
oauth_request.sign_request(signature_method_hmac_sha1, consumer, access_token)
connection = httplib.HTTPConnection(DASHBOARD_SERVER)
connection.request('GET', request_string)
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
connection.request('POST', ADD_COVERAGE_DATA_URL,
body=oauth_request.to_postdata(),
headers=headers)
response = connection.getresponse()
if response.status != 200:
message = ('Error: Failed to report to %s%s: got response %d (%s)' %
@ -96,6 +155,9 @@ def _report_coverage_to_dashboard(now, line_coverage, function_coverage):
def _main():
access_token = _read_access_token_from_file('access.token')
customer_secret = _read_customer_secret_from_file('customer.secret')
coverage_www_dir = os.path.join('/home', BUILD_BOT_USER, 'www')
www_dir_contents = os.listdir(coverage_www_dir)
@ -114,7 +176,8 @@ def _main():
function_coverage = _grab_coverage_percentage('Functions:', whole_file)
now = int(time.time())
_report_coverage_to_dashboard(now, line_coverage, function_coverage)
_report_coverage_to_dashboard(now, line_coverage, function_coverage,
access_token, CONSUMER_KEY, customer_secret)
if __name__ == '__main__':
_main()