From 44eb87e6dd3ddf9711d480521e006ce9084fdbd6 Mon Sep 17 00:00:00 2001 From: "andresp@webrtc.org" Date: Mon, 17 Mar 2014 14:23:22 +0000 Subject: [PATCH] Tool to establish a loopback call via apprtc turn server. For now the test keeps track of video bandwidth estimation and plots it using google visualization libraries after the test is concluded. There is also scripts to run a test and record the tcpdump. BUG=3037 R=hta@webrtc.org, phoglund@webrtc.org Review URL: https://webrtc-codereview.appspot.com/9729004 git-svn-id: http://webrtc.googlecode.com/svn/trunk@5707 4adac7df-926f-26a2-2b94-8c16560cd09d --- webrtc/tools/loopback_test/OWNERS | 1 + webrtc/tools/loopback_test/README | 12 + webrtc/tools/loopback_test/adapter.js | 211 ++++++++++++++++ webrtc/tools/loopback_test/loopback_test.html | 220 +++++++++++++++++ webrtc/tools/loopback_test/loopback_test.js | 232 ++++++++++++++++++ webrtc/tools/loopback_test/record-test.sh | 60 +++++ webrtc/tools/loopback_test/run-server.sh | 15 ++ webrtc/tools/loopback_test/stat_tracker.js | 94 +++++++ 8 files changed, 845 insertions(+) create mode 100644 webrtc/tools/loopback_test/OWNERS create mode 100644 webrtc/tools/loopback_test/README create mode 100644 webrtc/tools/loopback_test/adapter.js create mode 100644 webrtc/tools/loopback_test/loopback_test.html create mode 100644 webrtc/tools/loopback_test/loopback_test.js create mode 100755 webrtc/tools/loopback_test/record-test.sh create mode 100755 webrtc/tools/loopback_test/run-server.sh create mode 100644 webrtc/tools/loopback_test/stat_tracker.js diff --git a/webrtc/tools/loopback_test/OWNERS b/webrtc/tools/loopback_test/OWNERS new file mode 100644 index 000000000..296f71fff --- /dev/null +++ b/webrtc/tools/loopback_test/OWNERS @@ -0,0 +1 @@ +andresp@webrtc.org diff --git a/webrtc/tools/loopback_test/README b/webrtc/tools/loopback_test/README new file mode 100644 index 000000000..68f8eed68 --- /dev/null +++ b/webrtc/tools/loopback_test/README @@ -0,0 +1,12 @@ +Loopback test + +This is a simple html test framework to run a loopback test which can go via +turn. For now the test is used to analyse bandwidth estimation and get records +for bad scenarios. + +How to run: + ./run-server.sh (to start python serving the tests) + Access http://localhost:8080/loopback_test.html to run the test + +How to record: + You can use record-test.sh to get a tcpdump of a test run. diff --git a/webrtc/tools/loopback_test/adapter.js b/webrtc/tools/loopback_test/adapter.js new file mode 100644 index 000000000..6c2bd04d4 --- /dev/null +++ b/webrtc/tools/loopback_test/adapter.js @@ -0,0 +1,211 @@ +/** + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file is cloned from samples/js/base/adapter.js +// Modify the original and do new copy instead of doing changes here. + +var RTCPeerConnection = null; +var getUserMedia = null; +var attachMediaStream = null; +var reattachMediaStream = null; +var webrtcDetectedBrowser = null; +var webrtcDetectedVersion = null; + +function trace(text) { + // This function is used for logging. + if (text[text.length - 1] == '\n') { + text = text.substring(0, text.length - 1); + } + console.log((performance.now() / 1000).toFixed(3) + ": " + text); +} +function maybeFixConfiguration(pcConfig) { + if (pcConfig == null) { + return; + } + for (var i = 0; i < pcConfig.iceServers.length; i++) { + if (pcConfig.iceServers[i].hasOwnProperty('urls')){ + pcConfig.iceServers[i]['url'] = pcConfig.iceServers[i]['urls']; + delete pcConfig.iceServers[i]['urls']; + } + } +} + +if (navigator.mozGetUserMedia) { + console.log("This appears to be Firefox"); + + webrtcDetectedBrowser = "firefox"; + + webrtcDetectedVersion = + parseInt(navigator.userAgent.match(/Firefox\/([0-9]+)\./)[1], 10); + + // The RTCPeerConnection object. + var RTCPeerConnection = function(pcConfig, pcConstraints) { + // .urls is not supported in FF yet. + maybeFixConfiguration(pcConfig); + return new mozRTCPeerConnection(pcConfig, pcConstraints); + } + + // The RTCSessionDescription object. + RTCSessionDescription = mozRTCSessionDescription; + + // The RTCIceCandidate object. + RTCIceCandidate = mozRTCIceCandidate; + + // Get UserMedia (only difference is the prefix). + // Code from Adam Barth. + getUserMedia = navigator.mozGetUserMedia.bind(navigator); + navigator.getUserMedia = getUserMedia; + + // Creates iceServer from the url for FF. + createIceServer = function(url, username, password) { + var iceServer = null; + var url_parts = url.split(':'); + if (url_parts[0].indexOf('stun') === 0) { + // Create iceServer with stun url. + iceServer = { 'url': url }; + } else if (url_parts[0].indexOf('turn') === 0) { + if (webrtcDetectedVersion < 27) { + // Create iceServer with turn url. + // Ignore the transport parameter from TURN url for FF version <=27. + var turn_url_parts = url.split("?"); + // Return null for createIceServer if transport=tcp. + if (turn_url_parts.length === 1 || + turn_url_parts[1].indexOf('transport=udp') === 0) { + iceServer = {'url': turn_url_parts[0], + 'credential': password, + 'username': username}; + } + } else { + // FF 27 and above supports transport parameters in TURN url, + // So passing in the full url to create iceServer. + iceServer = {'url': url, + 'credential': password, + 'username': username}; + } + } + return iceServer; + }; + + createIceServers = function(urls, username, password) { + var iceServers = []; + // Use .url for FireFox. + for (i = 0; i < urls.length; i++) { + var iceServer = createIceServer(urls[i], + username, + password); + if (iceServer !== null) { + iceServers.push(iceServer); + } + } + return iceServers; + } + + // Attach a media stream to an element. + attachMediaStream = function(element, stream) { + console.log("Attaching media stream"); + element.mozSrcObject = stream; + element.play(); + }; + + reattachMediaStream = function(to, from) { + console.log("Reattaching media stream"); + to.mozSrcObject = from.mozSrcObject; + to.play(); + }; + + // Fake get{Video,Audio}Tracks + if (!MediaStream.prototype.getVideoTracks) { + MediaStream.prototype.getVideoTracks = function() { + return []; + }; + } + + if (!MediaStream.prototype.getAudioTracks) { + MediaStream.prototype.getAudioTracks = function() { + return []; + }; + } +} else if (navigator.webkitGetUserMedia) { + console.log("This appears to be Chrome"); + + webrtcDetectedBrowser = "chrome"; + webrtcDetectedVersion = + parseInt(navigator.userAgent.match(/Chrom(e|ium)\/([0-9]+)\./)[2], 10); + + // Creates iceServer from the url for Chrome M33 and earlier. + createIceServer = function(url, username, password) { + var iceServer = null; + var url_parts = url.split(':'); + if (url_parts[0].indexOf('stun') === 0) { + // Create iceServer with stun url. + iceServer = { 'url': url }; + } else if (url_parts[0].indexOf('turn') === 0) { + // Chrome M28 & above uses below TURN format. + iceServer = {'url': url, + 'credential': password, + 'username': username}; + } + return iceServer; + }; + + // Creates iceServers from the urls for Chrome M34 and above. + createIceServers = function(urls, username, password) { + var iceServers = []; + if (webrtcDetectedVersion >= 34) { + // .urls is supported since Chrome M34. + iceServers = {'urls': urls, + 'credential': password, + 'username': username }; + } else { + for (i = 0; i < urls.length; i++) { + var iceServer = createIceServer(urls[i], + username, + password); + if (iceServer !== null) { + iceServers.push(iceServer); + } + } + } + return iceServers; + }; + + // The RTCPeerConnection object. + var RTCPeerConnection = function(pcConfig, pcConstraints) { + // .urls is supported since Chrome M34. + if (webrtcDetectedVersion < 34) { + maybeFixConfiguration(pcConfig); + } + return new webkitRTCPeerConnection(pcConfig, pcConstraints); + } + + // Get UserMedia (only difference is the prefix). + // Code from Adam Barth. + getUserMedia = navigator.webkitGetUserMedia.bind(navigator); + navigator.getUserMedia = getUserMedia; + + // Attach a media stream to an element. + attachMediaStream = function(element, stream) { + if (typeof element.srcObject !== 'undefined') { + element.srcObject = stream; + } else if (typeof element.mozSrcObject !== 'undefined') { + element.mozSrcObject = stream; + } else if (typeof element.src !== 'undefined') { + element.src = URL.createObjectURL(stream); + } else { + console.log('Error attaching stream to element.'); + } + }; + + reattachMediaStream = function(to, from) { + to.src = from.src; + }; +} else { + console.log("Browser does not appear to be WebRTC-capable"); +} diff --git a/webrtc/tools/loopback_test/loopback_test.html b/webrtc/tools/loopback_test/loopback_test.html new file mode 100644 index 000000000..ea3fc2a8d --- /dev/null +++ b/webrtc/tools/loopback_test/loopback_test.html @@ -0,0 +1,220 @@ + + + + +Loopback test + + + + + + + + + + + + + + + + +
+

Duration (s):

+

Max video bitrate (kbps):

+

Force TURN:

+

+

+
+
+
+
+
+
+ + + diff --git a/webrtc/tools/loopback_test/loopback_test.js b/webrtc/tools/loopback_test/loopback_test.js new file mode 100644 index 000000000..b99204d1a --- /dev/null +++ b/webrtc/tools/loopback_test/loopback_test.js @@ -0,0 +1,232 @@ +/** + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// LoopbackTest establish a one way loopback call between 2 peer connections +// while continuously monitoring bandwidth stats. The idea is to use this as +// a base for other future tests and to keep track of more than just bandwidth +// stats. +// +// Usage: +// var test = new LoopbackTest(stream, callDurationMs, +// forceTurn, maxVideoBitrateKbps); +// test.run(onDone); +// function onDone() { +// test.getResults(); // return stats recorded during the loopback test. +// } +// +function LoopbackTest(stream, callDurationMs, forceTurn, maxVideoBitrateKbps) { + var pc1StatTracker; + var pc2StatTracker; + + // In order to study effect of network (e.g. wifi) on peer connection one can + // establish a loopback call and force it to go via a turn server. This way + // the call won't switch to local addresses. That is achieved by filtering out + // all non-relay ice candidades on both peers. + function constrainTurnCandidates(pc) { + var origAddIceCandidate = pc.addIceCandidate; + pc.addIceCandidate = function (candidate, successCallback, + failureCallback) { + if (forceTurn && candidate.candidate.indexOf("typ relay ") == -1) { + trace("Dropping non-turn candidate: " + candidate.candidate); + successCallback(); + return; + } else { + origAddIceCandidate.call(this, candidate, successCallback, + failureCallback); + } + } + } + + // FEC makes it hard to study bwe estimation since there seems to be a spike + // when it is enabled and disabled. Disable it for now. FEC issue tracked on: + // https://code.google.com/p/webrtc/issues/detail?id=3050 + function constrainOfferToRemoveFec(pc) { + var origCreateOffer = pc.createOffer; + pc.createOffer = function (successCallback, failureCallback, options) { + function filteredSuccessCallback(desc) { + desc.sdp = desc.sdp.replace(/(m=video 1 [^\r]+)(116 117)(\r\n)/g, + '$1\r\n'); + desc.sdp = desc.sdp.replace(/a=rtpmap:116 red\/90000\r\n/g, ''); + desc.sdp = desc.sdp.replace(/a=rtpmap:117 ulpfec\/90000\r\n/g, ''); + successCallback(desc); + } + origCreateOffer.call(this, filteredSuccessCallback, failureCallback, + options); + } + } + + // Constraint max video bitrate by modifying the SDP when creating an answer. + function constrainBitrateAnswer(pc) { + var origCreateAnswer = pc.createAnswer; + pc.createAnswer = function (successCallback, failureCallback, options) { + function filteredSuccessCallback(desc) { + if (maxVideoBitrateKbps) { + desc.sdp = desc.sdp.replace( + /a=mid:video\r\n/g, + 'a=mid:video\r\nb=AS:' + maxVideoBitrateKbps + '\r\n'); + } + successCallback(desc); + } + origCreateAnswer.call(this, filteredSuccessCallback, failureCallback, + options); + } + } + + // Run the actual LoopbackTest. + this.run = function(doneCallback) { + if (forceTurn) requestTurn(start, fail); + else start(); + + function start(turnServer) { + var pcConfig = forceTurn ? { iceServers: [turnServer] } : null; + console.log(pcConfig); + var pc1 = new RTCPeerConnection(pcConfig); + constrainTurnCandidates(pc1); + constrainOfferToRemoveFec(pc1); + pc1StatTracker = new StatTracker(pc1, 50); + pc1StatTracker.recordStat("EstimatedSendBitrate", + "bweforvideo", "googAvailableSendBandwidth"); + pc1StatTracker.recordStat("TransmitBitrate", + "bweforvideo", "googTransmitBitrate"); + pc1StatTracker.recordStat("TargetEncodeBitrate", + "bweforvideo", "googTargetEncBitrate"); + pc1StatTracker.recordStat("ActualEncodedBitrate", + "bweforvideo", "googActualEncBitrate"); + + var pc2 = new RTCPeerConnection(pcConfig); + constrainTurnCandidates(pc2); + constrainBitrateAnswer(pc2); + pc2StatTracker = new StatTracker(pc2, 50); + pc2StatTracker.recordStat("REMB", + "bweforvideo", "googAvailableReceiveBandwidth"); + + pc1.addStream(stream); + var call = new Call(pc1, pc2); + + call.start(); + setTimeout(function () { + call.stop(); + pc1StatTracker.stop(); + pc2StatTracker.stop(); + success(); + }, callDurationMs); + } + + function success() { + trace("Success"); + doneCallback(); + } + + function fail() { + trace("Fail"); + doneCallback(); + } + } + + // Returns a google visualization datatable with the recorded samples during + // the loopback test. + this.getResults = function () { + return mergeDataTable(pc1StatTracker.dataTable(), + pc2StatTracker.dataTable()); + } + + // Helper class to establish and manage a call between 2 peer connections. + // Usage: + // var c = new Call(pc1, pc2); + // c.start(); + // c.stop(); + // + function Call(pc1, pc2) { + pc1.onicecandidate = applyIceCandidate.bind(pc2); + pc2.onicecandidate = applyIceCandidate.bind(pc1); + + function applyIceCandidate(e) { + if (e.candidate) { + this.addIceCandidate(new RTCIceCandidate(e.candidate), + onAddIceCandidateSuccess, + onAddIceCandidateError); + } + } + + function onAddIceCandidateSuccess() {} + function onAddIceCandidateError(error) { + trace("Failed to add Ice Candidate: " + error.toString()); + } + + this.start = function() { + pc1.createOffer(gotDescription1, onCreateSessionDescriptionError); + + function onCreateSessionDescriptionError(error) { + trace('Failed to create session description: ' + error.toString()); + } + + function gotDescription1(desc){ + trace("Offer: " + desc.sdp); + pc1.setLocalDescription(desc); + pc2.setRemoteDescription(desc); + // Since the "remote" side has no media stream we need + // to pass in the right constraints in order for it to + // accept the incoming offer of audio and video. + pc2.createAnswer(gotDescription2, onCreateSessionDescriptionError); + } + + function gotDescription2(desc){ + trace("Answer: " + desc.sdp); + pc2.setLocalDescription(desc); + pc1.setRemoteDescription(desc); + } + } + + this.stop = function() { + pc1.close(); + pc2.close(); + } + } + + // Request a turn server. This uses the same servers as apprtc. + function requestTurn(successCallback, failureCallback) { + var currentDomain = document.domain; + if (currentDomain.search('localhost') === -1 && + currentDomain.search('apprtc') === -1) { + onerror("not authorized domain"); + return; + } + + // Get a turn server from computeengineondemand.appspot.com. + var turnUrl = 'https://computeengineondemand.appspot.com/' + + 'turn?username=156547625762562&key=4080218913'; + var xmlhttp = new XMLHttpRequest(); + xmlhttp.onreadystatechange = onTurnResult; + xmlhttp.open('GET', turnUrl, true); + xmlhttp.send(); + + function onTurnResult() { + if (this.readyState !== 4) { + return; + } + + if (this.status === 200) { + var turnServer = JSON.parse(xmlhttp.responseText); + // Create turnUris using the polyfill (adapter.js). + turnServer.uris = turnServer.uris.filter( + function (e) { return e.search('transport=udp') != -1; } + ); + var iceServers = createIceServers(turnServer.uris, + turnServer.username, + turnServer.password); + if (iceServers !== null) { + successCallback(iceServers); + return; + } + } + failureCallback("Failed to get a turn server."); + } + } +} diff --git a/webrtc/tools/loopback_test/record-test.sh b/webrtc/tools/loopback_test/record-test.sh new file mode 100755 index 000000000..92d920243 --- /dev/null +++ b/webrtc/tools/loopback_test/record-test.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# +# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +# +# This script is used to record a tcp dump of running a loop back test. +# Example use case: +# +# $ ./run-server.sh & # spawns a server to serve the html pages +# # on localhost:8080 +# +# (recording 3 tests with 5mins and bitrates 1mbps, 2mbps and 3mbps) +# $ sudo -v # Caches sudo credentials needed +# # for tcpdump +# $ export INTERFACE=eth1 # Defines interface to record packets +# $ export CHROME_UNDER_TESTING=./chrome # Define which chrome to run on tests +# $ export TEST="http://localhost:8080/loopback_test.html?auto-mode=true" +# $ record-test.sh ./record1.pcap "$TEST&duration=300&max-video-bitrate=1000" +# $ record-test.sh ./record2.pcap "$TEST&duration=300&max-video-bitrate=2000" +# $ record-test.sh ./record3.pcap "$TEST&duration=300&max-video-bitrate=3000" + +# Indicate an error and exit with a nonzero status if any of the required +# environment variables is Null or Unset. +: ${INTERFACE:?"Need to set INTERFACE env variable"} +: ${CHROME_UNDER_TESTING:?"Need to set CHROME_UNDER_TESTING env variable"} + +if [ ! -x "$CHROME_UNDER_TESTING" ]; then + echo "CHROME_UNDER_TESTING=$CHROME_UNDER_TESTING does not seem to exist." + exit 1 +fi + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi +TEST_URL=$1 +OUTPUT_RECORDING=$2 + +sudo -nv > /dev/null 2>&1 +if [ $? != 0 ]; then + echo "Run \"sudo -v\" to cache your credentials." \ + "They are needed to run tcpdump." + exit +fi + +echo "Recording $INTERFACE into ${OUTPUT_RECORDING}" +sudo -n tcpdump -i "$INTERFACE" -w - > "${OUTPUT_RECORDING}" & +TCPDUMP_PID=$! + +echo "Starting ${CHROME_UNDER_TESTING} with ${TEST_URL}." +# Using real camera instead of --use-fake-device-for-media-stream as it +# does not produces images complex enough to reach 3mbps. +# Flag --use-fake-ui-for-media-stream automatically allows getUserMedia calls. +$CHROME_UNDER_TESTING --use-fake-ui-for-media-stream "${TEST_URL}" +kill ${TCPDUMP_PID} diff --git a/webrtc/tools/loopback_test/run-server.sh b/webrtc/tools/loopback_test/run-server.sh new file mode 100755 index 000000000..35c0797c2 --- /dev/null +++ b/webrtc/tools/loopback_test/run-server.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# +# Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +# +# This script is used to launch a simple http server for files in the same +# location as the script itself. +cd "`dirname \"$0\"`" +echo "Starting http server in port 8080." +exec python -m SimpleHTTPServer 8080 diff --git a/webrtc/tools/loopback_test/stat_tracker.js b/webrtc/tools/loopback_test/stat_tracker.js new file mode 100644 index 000000000..49f46c39f --- /dev/null +++ b/webrtc/tools/loopback_test/stat_tracker.js @@ -0,0 +1,94 @@ +/** + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// StatTracker is a helper class to keep track of stats on a RTCPeerConnection +// object. It uses google visualization datatables to keep the recorded samples +// and simplify plugging them into graphs later. +// +// Usage example: +// var tracker = new StatTracker(pc, pollInterval); +// tracker.recordStat("EstimatedSendBitrate", +// "bweforvideo", "googAvailableSendBandwidth"); +// ... +// tracker.stop(); +// tracker.dataTable(); // returns the recorded values. In this case +// a table with 2 columns { Time, EstimatedSendBitrate } and a row for each +// sample taken until stop() was called. +// +function StatTracker(pc, pollInterval) { + pollInterval = pollInterval || 250; + + var dataTable = new google.visualization.DataTable(); + var timeColumnIndex = dataTable.addColumn('datetime', 'Time'); + var recording = true; + + // Set of sampling functions. Functions registered here are called + // once per getStats with the given report and a rowIndex for the + // sample period so they can extract and record the tracked variables. + var samplingFunctions = {}; + + // Accessor to the current recorded stats. + this.dataTable = function() { return dataTable; } + + // recordStat(varName, recordName, statName) adds a samplingFunction that + // records namedItem(recordName).stat(statName) from RTCStatsReport for each + // sample into a column named varName in the dataTable. + this.recordStat = function (varName, recordName, statName) { + var columnIndex = dataTable.addColumn('number', varName); + samplingFunctions[varName] = function (report, rowIndex) { + var sample; + var record = report.namedItem(recordName); + if (record) sample = record.stat(statName); + dataTable.setCell(rowIndex, columnIndex, sample); + } + } + + // Stops the polling of stats from the peer connection. + this.stop = function() { + recording = false; + } + + // RTCPeerConnection.getStats is asynchronous. In order to avoid having + // too many pending getStats requests going, this code only queues the + // next getStats with setTimeout after the previous one returns, instead + // of using setInterval. + function poll() { + pc.getStats(function (report) { + if (!recording) return; + setTimeout(poll, pollInterval); + var result = report.result(); + if (result.length < 1) return; + + var rowIndex = dataTable.addRow(); + dataTable.setCell(rowIndex, timeColumnIndex, result[0].timestamp); + for (var v in samplingFunctions) + samplingFunctions[v](report, rowIndex); + }); + } + setTimeout(poll, pollInterval); +} + +/** + * Utility method to perform a full join between data tables from StatTracker. + */ +function mergeDataTable(dataTable1, dataTable2) { + function allColumns(cols) { + var a = []; + for (var i = 1; i < cols; ++i) a.push(i); + return a; + } + return google.visualization.data.join( + dataTable1, + dataTable2, + 'full', + [[0, 0]], + allColumns(dataTable1.getNumberOfColumns()), + allColumns(dataTable2.getNumberOfColumns())); +}