Updated conformance tests and w3c-ified them.

I intend here to put these up for review on W3C. This moves the tests
to use the W3C-style vendor prefix handling and updates the tests to
the latest drafts.

This yields 44 Pass 24 Fail and 13 pass 54 fail 1 timeout on Firefox.
As far I can tell all failures are correct; in particular FF media
media stream tracks do not adhere to the standard.

Also I can't get FF to get a remote video up in the peerconnection
test, just the local one.

BUG=webrtc:3455
R=kjellander@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/14639004

git-svn-id: http://webrtc.googlecode.com/svn/trunk@6370 4adac7df-926f-26a2-2b94-8c16560cd09d

webrtc/test/w3c/getusermedia_conformance_test.html

@@ -27,9 +27,9 @@ Notice that this requires the site you're browsing to use HTTPS.
   <p>This page contains a foundation of conformance tests that can be expanded
   to cover most things in the W3C specification of the Media Capture and Streams
   API.</p>
-  <p>VERSION: These tests are based on the W3C Editor's Draft of August 24th,
+  <p>VERSION: These tests are based on the W3C Editor's Draft of 07 May,
-  2013
+  2014
-  (<a href="http://dev.w3.org/2011/webrtc/editor/archives/20130824/getusermedia.html">http://dev.w3.org/2011/webrtc/editor/archives/20130824/getusermedia.html</a>)
+  (<a href="http://dev.w3.org/2011/webrtc/editor/archives/20140507/getusermedia.html">http://dev.w3.org/2011/webrtc/editor/archives/20140507/getusermedia.html</a>)
   <p>STATUS: In its current state, it only performs simple checks on the various
   attributes and methods of the objects exposed by the API. There's not much
   functionality tested so far. The spec doesn't define if an attribute shall be
@@ -37,9 +37,7 @@ Notice that this requires the site you're browsing to use HTTPS.
   inherited (<code>assert_inherits</code>). Since testharness.js doesn't offer
   any generic function that covers both, the method for verification is
   currently chosen according to the current Chrome implementation.</p>
-  <p>PREFIXES: These tests currently utilizes the <code>adapter.js</code>
+  <p>HOW TO RUN: For Chrome, do this:
-  script, which handle the prefixes used by different browsers.</p>
-  <p>HOW TO RUN: The easiest way is to tell your browser to:
   <ul>
   <li>Provide a fake webcam (<code>--use-fake-ui-for-media-stream</code> in
   Chrome)</li>
@@ -49,14 +47,20 @@ Notice that this requires the site you're browsing to use HTTPS.
   (<code>--allow-file-access-from-files</code> in Chrome)</li>
   </ul>
   Then just load this HTML file to execute the tests.</p>
+  <p>For Firefox, set the media.navigator.permission.disabled property to true
+  about:config. You must have a webcam and microphone available on the system.
+  </p>
 
   <div id="log"></div>
-  <video width="320" height="240" id="local-view" autoplay="autoplay"
+  <video id="local-view" autoplay="autoplay" muted="true"></video>
-      muted="true"></video>
 
-  <script src="https://w3c-test.org/resources/testharness.js"></script>
+  <!-- These files are in place when executing on W3C. -->
-  <!-- Load the polyfill to switch-hit between Chrome and Firefox -->
+  <script src="/resources/testharness.js"></script>
-  <script src="https://webrtc.googlecode.com/svn/stable/webrtc/test/manual/adapter.js"></script>
+  <script src="/resources/testharnessreport.js"></script>
+  <script src="/common/vendor-prefix.js"
+          data-prefixed-objects=   '[{"ancestors":["navigator"], "name":"getUserMedia"},
+                                     {"ancestors":["window"], "name":"RTCPeerConnection"}]'
+          data-prefixed-prototypes='[{"ancestors":["HTMLMediaElement"],"name":"srcObject"}]'></script>
 
   <script src="getusermedia_conformance_test.js"></script>
 </body>

webrtc/test/w3c/getusermedia_conformance_test.js

@@ -15,7 +15,7 @@ function failedCallback(test) {
   });
 }
 function invokeGetUserMedia(test, okCallback) {
-  getUserMedia({ video: true, audio: true }, okCallback,
+  navigator.getUserMedia({ video: true, audio: true }, okCallback,
       failedCallback(test));
 }
 
@@ -105,7 +105,7 @@ mediaStreamTest.step(function() {
     // TODO(kjellander): verify number of tracks.
     mediaStreamTest.done();
   });
-  invokeGetUserMedia(mediaStreamTest, okCallback);;
+  invokeGetUserMedia(mediaStreamTest, okCallback);
 });
 
 // 4.3 MediaStreamTrack.
@@ -158,14 +158,12 @@ function verifyTrack(type, track) {
   }, '[MediaStreamTrack (' + type + ')] onunmute EventHandler');
 
   test(function () {
-    // Missing in Chrome.
     assert_own_property(track, '_readonly');
     assert_readonly(track, '_readonly');
     assert_true(typeof track._readonly === 'boolean');
   }, '[MediaStreamTrack (' + type + ')] _readonly attribute');
 
   test(function () {
-    // Missing in Chrome.
     assert_own_property(track, 'remote');
     assert_readonly(track, 'remote');
     assert_true(typeof track.remote === 'boolean');
@@ -179,7 +177,6 @@ function verifyTrack(type, track) {
   }, '[MediaStreamTrack (' + type + ')] readyState attribute');
 
   test(function () {
-    // Missing in Chrome.
     assert_own_property(track, 'onstarted');
     assert_true(track.onstarted === null);
   }, '[MediaStreamTrack (' + type + ')] onstarted EventHandler');
@@ -190,53 +187,41 @@ function verifyTrack(type, track) {
   }, '[MediaStreamTrack (' + type + ')] onended EventHandler');
 
   test(function () {
-    // Missing in Chrome.
+    assert_inherits(track, 'getNativeSettings');
-    assert_inherits(track, 'getSourceInfos');
-    assert_true(typeof track.getSourceInfos === 'function');
-  }, '[MediaStreamTrack (' + type + ')]: getSourceInfos function');
-
-  test(function () {
-    // Missing in Chrome.
-    assert_inherits(track, 'constraints');
-    assert_true(typeof track.constraints === 'function');
-  }, '[MediaStreamTrack (' + type + ')]: constraints function');
-
-  test(function () {
-    // Missing in Chrome.
-    assert_inherits(track, 'states');
-    assert_true(typeof track.states === 'function');
-  }, '[MediaStreamTrack (' + type + ')]: states function');
-
-  test(function () {
-    // Missing in Chrome.
-    assert_inherits(track, 'capabilities');
     assert_true(typeof track.capabilities === 'function');
-  }, '[MediaStreamTrack (' + type + ')]: capabilities function');
+  }, '[MediaStreamTrack (' + type + ')]: getNativeSettings function');
 
   test(function () {
-    // Missing in Chrome.
-    assert_inherits(track, 'applyConstraints');
-    assert_true(typeof track.applyConstraints === 'function');
-  }, '[MediaStreamTrack (' + type + ')]: applyConstraints function');
-
-  test(function () {
-    // Missing in Chrome.
-    assert_own_property(track, 'onoverconstrained');
-    assert_true(track.onoverconstrained === null);
-  }, '[MediaStreamTrack (' + type + ')] onoverconstrained EventHandler');
-
-  test(function () {
-    // Missing in Chrome.
     assert_inherits(track, 'clone');
     assert_true(typeof track.clone === 'function');
   }, '[MediaStreamTrack (' + type + ')] clone function');
 
   test(function () {
-    // Missing in Chrome.
     assert_inherits(track, 'stop');
     assert_true(typeof track.stop === 'function');
   }, '[MediaStreamTrack (' + type + ')] stop function');
+
+  test(function () {
+    assert_inherits(track, 'getCapabilities');
+    assert_true(typeof track.capabilities === 'function');
+  }, '[MediaStreamTrack (' + type + ')]: getCapabilities function');
+
+  test(function () {
+    assert_inherits(track, 'getConstraints');
+    assert_true(typeof track.constraints === 'function');
+  }, '[MediaStreamTrack (' + type + ')]: getConstraints function');
+
+  test(function () {
+    assert_inherits(track, 'getSettings');
+    assert_true(typeof track.constraints === 'function');
+  }, '[MediaStreamTrack (' + type + ')]: getSettings function');
+
+  test(function () {
+    assert_inherits(track, 'applyConstraints');
+    assert_true(typeof track.applyConstraints === 'function');
+  }, '[MediaStreamTrack (' + type + ')]: applyConstraints function');
 };
+
 mediaStreamTrackTest.step(function() {
   var okCallback = mediaStreamTrackTest.step_func(function (stream) {
     verifyTrack('audio', stream.getAudioTracks()[0]);
@@ -250,11 +235,11 @@ mediaStreamTrackTest.step(function() {
   var okCallback = mediaStreamTrackTest.step_func(function (stream) {
     // Verify event handlers are working.
     var track = stream.getVideoTracks()[0];
-    track.onended = onendedCallback
+    track.onended = onEndedCallback
     track.stop();
     mediaStreamTrackTest.done();
   });
-  var onendedCallback = mediaStreamTrackTest.step_func(function () {
+  var onEndedCallback = mediaStreamTrackTest.step_func(function () {
     assert_true(track.ended);
     mediaStreamTrackTest.done();
   });
@@ -296,7 +281,7 @@ avTracksTest.step(function() {
 var createObjectURLTest = async_test('8.1 URL createObjectURL method');
 createObjectURLTest.step(function() {
   var okCallback = createObjectURLTest.step_func(function (stream) {
-    var url = webkitURL.createObjectURL(stream);
+    var url = URL.createObjectURL(stream);
     assert_true(typeof url === 'string');
     createObjectURLTest.done();
   });
@@ -363,8 +348,7 @@ function verifyVideoTagWithStream(videoTag) {
 mediaElementsTest.step(function() {
   var okCallback = mediaElementsTest.step_func(function (stream) {
     var videoTag = document.getElementById('local-view');
-    // Call the polyfill wrapper to attach the media stream to this element.
+    videoTag.srcObject = stream;
-    attachMediaStream(videoTag, stream);
     verifyVideoTagWithStream(videoTag);
     mediaElementsTest.done();
   });
@@ -381,17 +365,12 @@ getUserMediaTest.step(function() {
     getUserMediaTest.done();
   });
 
-  // boolean parameters, without failure callback:
+  // All three arguments are mandatory, so pass all of them.
-  getUserMedia({ video: true, audio: true }, okCallback);
+  navigator.getUserMedia({ video: true, audio: true }, okCallback,
-  getUserMedia({ video: true, audio: false }, okCallback);
-  getUserMedia({ video: false, audio: true }, okCallback);
-
-  // boolean parameters, with failure callback:
-  getUserMedia({ video: true, audio: true }, okCallback,
       failedCallback(getUserMediaTest));
-  getUserMedia({ video: true, audio: false }, okCallback,
+  navigator.getUserMedia({ video: true, audio: false }, okCallback,
       failedCallback(getUserMediaTest));
-  getUserMedia({ video: false, audio: true }, okCallback,
+  navigator.getUserMedia({ video: false, audio: true }, okCallback,
       failedCallback(getUserMediaTest));
 });
 
@@ -413,7 +392,8 @@ constraintsTest.step(function() {
   constraints.video.mandatory.minHeight = 480;
   constraints.video.mandatory.minFrameRate = 15;
 
-  getUserMedia(constraints, okCallback, failedCallback(constraintsTest));
+  navigator.getUserMedia(constraints, okCallback,
+                         failedCallback(constraintsTest));
 });
 
 // 11.3 NavigatorUserMediaSuccessCallback.
@@ -427,25 +407,3 @@ successCallbackTest.step(function() {
   invokeGetUserMedia(successCallbackTest, okCallback);
 });
 
-// 11.4 NavigatorUserMediaError and NavigatorUserMediaErrorCallback.
-var errorCallbackTest = async_test('11.4 NavigatorUserMediaError and ' +
-                                   'NavigatorUserMediaErrorCallback');
-errorCallbackTest.step(function() {
-  var okCallback = errorCallbackTest.step_func(function (stream) {
-    assert_unreached('Should not get a success callback');
-  });
-  var errorCallback = errorCallbackTest.step_func(function (error) {
-    assert_own_property(error, 'name');
-    assert_readonly(error.name);
-    assert_true(typeof error.name === 'string');
-    assert_equals(error.name, 'ConstraintNotSatisfiedError', 'error.name');
-    errorCallbackTest.done();
-  });
-  // Setting both audio and video to false triggers an error callback.
-  // TODO(kjellander): Figure out if there's a way in the spec to trigger an
-  // error callback.
-
-  // TODO(kjellander): Investigate why the error callback is not called when
-  // false/false is provided in Chrome.
-  getUserMedia({ video: false, audio: false }, okCallback, errorCallback);
-});

webrtc/test/w3c/peerconnection_conformance_test.html

@@ -1,35 +1,51 @@
 <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
 
 <!--
-To quickly iterate when developing this test, make sure you select
+To quickly iterate when developing this test, use --use-fake-ui-for-media-stream
-'Always allow this site to use this webcam' option in the dropdown menu of
+for Chrome and set the media.navigator.permission.disabled property to true in
-Chrome when it's requesting access to your webcam.
+Firefox. You must either have a webcam/mic available on the system or use for
-Notice that this requires the site you're browsing to use HTTPS.
+instance --use-fake-device-for-media-stream for Chrome.
-
-Without that, the test might timeout before you have had the chance to accept
-access to the webcam.
 -->
 
 <html>
 <head>
   <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
   <title>PeerConnection Connection Test</title>
+</head>
+<body>
+  <div id="log"></div>
+  <div>
+    <video id="local-view" autoplay="autoplay"></video>
+    <video id="remote-view" autoplay="autoplay"/>
+    </video>
+  </div>
 
-  <script src="https://w3c-test.org/resources/testharness.js"></script>
+  <!-- These files are in place when executing on W3C. -->
-
+  <script src="/resources/testharness.js"></script>
+  <script src="/resources/testharnessreport.js"></script>
+  <script src="/common/vendor-prefix.js"
+          data-prefixed-objects=
+              '[{"ancestors":["navigator"], "name":"getUserMedia"},
+                {"ancestors":["window"], "name":"RTCPeerConnection"},
+                {"ancestors":["window"], "name":"RTCSessionDescription"},
+                {"ancestors":["window"], "name":"RTCIceCandidate"}]'
+          data-prefixed-prototypes=
+               '[{"ancestors":["HTMLMediaElement"],"name":"srcObject"}]'>
+  </script>
   <script type="text/javascript">
   var test = async_test('Can set up a basic WebRTC call.', {timeout: 5000});
+
   var gFirstConnection = null;
   var gSecondConnection = null;
 
   function getUserMediaOkCallback(localStream) {
-    gFirstConnection = new webkitRTCPeerConnection(null, null);
+    gFirstConnection = new RTCPeerConnection(null, null);
     gFirstConnection.onicecandidate = onIceCandidateToFirst;
     gFirstConnection.addStream(localStream);
-    gFirstConnection.createOffer(onOfferCreated);
+    gFirstConnection.createOffer(onOfferCreated, failed('createOffer'));
 
     var videoTag = document.getElementById('local-view');
-    videoTag.src = webkitURL.createObjectURL(localStream);
+    videoTag.src = URL.createObjectURL(localStream);
   };
 
   var onOfferCreated = test.step_func(function(offer) {
@@ -41,7 +57,7 @@ access to the webcam.
   });
 
   function receiveCall(offerSdp) {
-    gSecondConnection = new webkitRTCPeerConnection(null, null);
+    gSecondConnection = new RTCPeerConnection(null, null);
     gSecondConnection.onicecandidate = onIceCandidateToSecond;
     gSecondConnection.onaddstream = onRemoteStream;
 
@@ -89,7 +105,7 @@ access to the webcam.
 
   var onRemoteStream = test.step_func(function(event) {
     var videoTag = document.getElementById('remote-view');
-    videoTag.src = webkitURL.createObjectURL(event.stream);
+    videoTag.src = URL.createObjectURL(event.stream);
   });
 
   // Returns a suitable error callback.
@@ -101,19 +117,11 @@ access to the webcam.
 
   // This function starts the test.
   test.step(function() {
-    navigator.webkitGetUserMedia({ video: true, audio: true },
+    navigator.getUserMedia({ video: true, audio: true },
         getUserMediaOkCallback,
         failed('getUserMedia'));
   });
 </script>
-</head>
 
-<body>
-
-<div>
-  <video width="320" height="240" id="remote-view" autoplay="autoplay"></video>
-  <video width="320" height="240" id="local-view" autoplay="autoplay"></video>
-</div>
-<div id="log"></div>
 </body>
 </html>