Adds two full stack performance metrics for end-to-end delay.

Review URL: https://webrtc-codereview.appspot.com/937034

git-svn-id: http://webrtc.googlecode.com/svn/trunk@3235 4adac7df-926f-26a2-2b94-8c16560cd09d

webrtc/video_engine/test/auto_test/automated/vie_video_verification_test.cc

@@ -116,25 +116,28 @@ class ParameterizedFullStackTest : public ViEVideoVerificationTest,
  protected:
   struct TestParameters {
     int packet_loss_rate;
-    int round_trip_time;
+    int one_way_delay;
     int bitrate;
     double avg_psnr_threshold;
     double avg_ssim_threshold;
+    std::string test_label;
   };
 
   void SetUp() {
     int i = 0;
     parameter_table_[i].packet_loss_rate = 0;
-    parameter_table_[i].round_trip_time = 0;
+    parameter_table_[i].one_way_delay = 0;
     parameter_table_[i].bitrate = 300;
     parameter_table_[i].avg_psnr_threshold = 35;
     parameter_table_[i].avg_ssim_threshold = 0.96;
+    parameter_table_[i].test_label = "net delay 0, plr 0";
     ++i;
     parameter_table_[i].packet_loss_rate = 5;
-    parameter_table_[i].round_trip_time = 50;
+    parameter_table_[i].one_way_delay = 50;
     parameter_table_[i].bitrate = 300;
     parameter_table_[i].avg_psnr_threshold = 35;
     parameter_table_[i].avg_ssim_threshold = 0.96;
+    parameter_table_[i].test_label = "net delay 50, plr 5";
   }
 
   TestParameters parameter_table_[2];
@@ -193,7 +196,7 @@ TEST_P(ParameterizedFullStackTest, RunsFullStackWithoutErrors)  {
   // frames every now and then.
   const int kBitRateKbps = parameter_table_[GetParam()].bitrate;
   const int kPacketLossPercent = parameter_table_[GetParam()].packet_loss_rate;
-  const int kNetworkDelayMs = parameter_table_[GetParam()].round_trip_time;
+  const int kNetworkDelayMs = parameter_table_[GetParam()].one_way_delay;
   int width = 352;
   int height = 288;
   ViETest::Log("Bit rate     : %5d kbps", kBitRateKbps);
@@ -207,7 +210,7 @@ TEST_P(ParameterizedFullStackTest, RunsFullStackWithoutErrors)  {
   StopRenderers();
 
   detector.CalculateResults();
-  detector.PrintReport();
+  detector.PrintReport(parameter_table_[GetParam()].test_label);
 
   if (detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kRendered) >
       detector.GetNumberOfFramesDroppedAt(FrameDropDetector::kDecoded)) {

webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.cc

@@ -9,6 +9,8 @@
  */
 
 #include <cassert>
+#include <cmath>
+#include <sstream>
 #include <string>
 
 #include "webrtc/modules/video_capture/include/video_capture_factory.h"
@@ -16,6 +18,7 @@
 #include "testsupport/fileutils.h"
 #include "testsupport/frame_reader.h"
 #include "testsupport/frame_writer.h"
+#include "testsupport/perf_test.h"
 #include "video_engine/test/auto_test/interface/vie_autotest.h"
 #include "video_engine/test/auto_test/interface/vie_autotest_defines.h"
 #include "video_engine/test/auto_test/primitives/framedrop_primitives.h"
@@ -99,6 +102,41 @@ class DecodedTimestampEffectFilter : public webrtc::ViEEffectFilter {
   FrameDropDetector* frame_drop_detector_;
 };
 
+class Statistics {
+ public:
+  Statistics() : sum_(0.0f), sum_squared_(0.0f), count_(0) {};
+
+  void AddSample(float sample) {
+    sum_ += sample;
+    sum_squared_ += sample * sample;
+    ++count_;
+  }
+
+  float Mean() {
+    if (count_ == 0)
+      return -1.0f;
+    return sum_ / count_;
+  }
+
+  float Variance() {
+    if (count_ == 0)
+      return -1.0f;
+    return  sum_squared_ / count_ - Mean() * Mean();
+  }
+
+  std::string AsString() {
+    std::stringstream ss;
+    ss << (Mean() >= 0 ? Mean() : -1) << ", " <<
+        (Variance() >= 0 ? sqrt(Variance()) : -1);
+    return ss.str();
+  }
+
+ private:
+  float sum_;
+  float sum_squared_;
+  int count_;
+};
+
 void TestFullStack(const TbInterfaces& interfaces,
                    int capture_id,
                    int video_channel,
@@ -338,7 +376,7 @@ void FrameDropDetector::CalculateResults() {
   dirty_ = false;
 }
 
-void FrameDropDetector::PrintReport() {
+void FrameDropDetector::PrintReport(const std::string& test_label) {
   assert(!dirty_);
   ViETest::Log("Frame Drop Detector report:");
   ViETest::Log("  Created  frames: %ld", created_frames_.size());
@@ -413,6 +451,8 @@ void FrameDropDetector::PrintReport() {
       "  latency");
   ViETest::Log("                                               (incl network)"
       "(excl network)");
+  Statistics latency_incl_network_stats;
+  Statistics latency_excl_network_stats;
   for (std::vector<Frame*>::const_iterator it = created_frames_vector_.begin();
        it != created_frames_vector_.end(); ++it) {
     int created_to_sent = (*it)->dropped_at_send ? -1 :
@@ -433,6 +473,12 @@ void FrameDropDetector::PrintReport() {
     int total_latency_excl_network = (*it)->dropped_at_render ? -1 :
         static_cast<int>((*it)->rendered_timestamp_in_us_ -
                          (*it)->created_timestamp_in_us_ - sent_to_received);
+    if (total_latency_incl_network >= 0)
+      latency_incl_network_stats.AddSample(total_latency_incl_network /
+                                           1000.0f);
+    if (total_latency_excl_network >= 0)
+      latency_excl_network_stats.AddSample(total_latency_excl_network /
+                                           1000.0f);
     ViETest::Log("%5d %9d %9d %9d %9d %12d %12d",
                  (*it)->number_,
                  created_to_sent,
@@ -442,6 +488,15 @@ void FrameDropDetector::PrintReport() {
                  total_latency_incl_network,
                  total_latency_excl_network);
   }
+
+  // Print dashboard data.
+  webrtc::test::PrintResultMeanAndError(
+      "total delay (excl. network)", " " + test_label, "",
+      latency_excl_network_stats.AsString(), "ms", false);
+  webrtc::test::PrintResultMeanAndError(
+      "total delay (incl. network)", " " + test_label, "",
+      latency_incl_network_stats.AsString(), "ms", false);
+
   // Find and print the dropped frames.
   ViETest::Log("\nTotal # dropped frames at:");
   ViETest::Log("  Send   : %d", dropped_frames_at_send_);

webrtc/video_engine/test/auto_test/primitives/framedrop_primitives.h

@@ -167,9 +167,11 @@ class FrameDropDetector {
   const std::vector<Frame*>& GetAllFrames();
 
   // Prints a detailed report about all the different frame states and which
-  // ones are detected as dropped, using ViETest::Log.
+  // ones are detected as dropped, using ViETest::Log. Also prints
+  // perf-formatted output and adds |test_label| as a modifier to the perf
+  // output.
   // CalculateResults() must be called before calling this method.
-  void PrintReport();
+  void PrintReport(const std::string& test_label);
 
   // Prints all the timestamp maps. Mainly used for debugging purposes to find
   // missing timestamps.