Add delay and send/receive throughput plots to BWE simulation.

R=mflodman@webrtc.org, solenberg@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/7999004

git-svn-id: http://webrtc.googlecode.com/svn/trunk@5491 4adac7df-926f-26a2-2b94-8c16560cd09d

webrtc/modules/modules.gyp

@@ -175,6 +175,7 @@
             'media_file/source/media_file_unittest.cc',
             'module_common_types_unittest.cc',
             'pacing/paced_sender_unittest.cc',
+            'remote_bitrate_estimator/bwe_simulations.cc',
             'remote_bitrate_estimator/include/mock/mock_remote_bitrate_observer.h',
             'remote_bitrate_estimator/rate_statistics_unittest.cc',
             'remote_bitrate_estimator/remote_bitrate_estimator_single_stream_unittest.cc',

webrtc/modules/remote_bitrate_estimator/bwe_simulations.cc

@@ -0,0 +1,87 @@
+/*
+ *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h"
+#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test.h"
+#include "webrtc/test/testsupport/fileutils.h"
+
+namespace webrtc {
+namespace testing {
+namespace bwe {
+#if BWE_TEST_LOGGING_COMPILE_TIME_ENABLE
+std::vector<BweTestConfig::EstimatorConfig> SingleEstimatorConfig() {
+  static const RemoteBitrateEstimatorFactory factory =
+      AbsoluteSendTimeRemoteBitrateEstimatorFactory();
+
+  std::vector<BweTestConfig::EstimatorConfig> result;
+  result.push_back(BweTestConfig::EstimatorConfig("AST", &factory));
+  return result;
+}
+
+std::vector<const PacketSenderFactory*> AdaptiveVideoSenderFactories(
+    uint32_t count) {
+  static const AdaptiveVideoPacketSenderFactory factories[] = {
+    AdaptiveVideoPacketSenderFactory(30.00f, 150, 0x1234, 0.13f),
+    AdaptiveVideoPacketSenderFactory(30.00f, 300, 0x3456, 0.26f),
+    AdaptiveVideoPacketSenderFactory(15.00f, 600, 0x4567, 0.39f),
+  };
+
+  assert(count <= sizeof(factories) / sizeof(factories[0]));
+
+  std::vector<const PacketSenderFactory*> result;
+  for (uint32_t i = 0; i < count; ++i) {
+    result.push_back(&factories[i]);
+  }
+  return result;
+}
+
+BweTestConfig MakeAdaptiveBweTestConfig(uint32_t sender_count) {
+  BweTestConfig result = {
+    AdaptiveVideoSenderFactories(sender_count), SingleEstimatorConfig()
+  };
+  return result;
+}
+
+// This test fixture is used to instantiate tests running with adaptive video
+// senders.
+class BweSimulation : public BweTest {
+ public:
+  BweSimulation() : BweTest() {}
+  virtual ~BweSimulation() {}
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(BweSimulation);
+};
+
+INSTANTIATE_TEST_CASE_P(VideoSendersTest, BweSimulation,
+    ::testing::Values(MakeAdaptiveBweTestConfig(1),
+                      MakeAdaptiveBweTestConfig(3)));
+
+TEST_P(BweSimulation, SprintUplinkTest) {
+  VerboseLogging(true);
+  RateCounterFilter counter1(this, "sender_output");
+  TraceBasedDeliveryFilter filter(this);
+  RateCounterFilter counter2(this, "receiver_input");
+  ASSERT_TRUE(filter.Init(test::ResourcePath("sprint-uplink", "rx")));
+  RunFor(60 * 1000);
+}
+
+TEST_P(BweSimulation, Verizon4gDownlinkTest) {
+  VerboseLogging(true);
+  RateCounterFilter counter1(this, "sender_output");
+  TraceBasedDeliveryFilter filter(this);
+  RateCounterFilter counter2(this, "receiver_input");
+  ASSERT_TRUE(filter.Init(test::ResourcePath("verizon4g-downlink", "rx")));
+  RunFor(22 * 60 * 1000);
+}
+#endif  // BWE_TEST_LOGGING_COMPILE_TIME_ENABLE
+}  // namespace bwe
+}  // namespace testing
+}  // namespace webrtc

webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc

@@ -15,38 +15,6 @@
 namespace webrtc {
 namespace testing {
 namespace bwe {
-
-class VideoPacketSenderFactory : public PacketSenderFactory {
- public:
-  VideoPacketSenderFactory(float fps, uint32_t kbps, uint32_t ssrc,
-                           float frame_offset)
-      : fps_(fps),
-        kbps_(kbps),
-        ssrc_(ssrc),
-        frame_offset_(frame_offset) {
-  }
-  virtual ~VideoPacketSenderFactory() {}
-  virtual PacketSender* Create() const {
-    return new VideoSender(NULL, fps_, kbps_, ssrc_, frame_offset_);
-  }
- protected:
-  float fps_;
-  uint32_t kbps_;
-  uint32_t ssrc_;
-  float frame_offset_;
-  bool adaptive_;
-};
-
-class AdaptiveVideoPacketSenderFactory : public VideoPacketSenderFactory {
- public:
-  AdaptiveVideoPacketSenderFactory(float fps, uint32_t kbps, uint32_t ssrc,
-                                   float frame_offset)
-      : VideoPacketSenderFactory(fps, kbps, ssrc, frame_offset) {}
-  virtual PacketSender* Create() const {
-    return new AdaptiveVideoSender(NULL, fps_, kbps_, ssrc_, frame_offset_);
-  }
-};
-
 std::vector<const PacketSenderFactory*> VideoSenderFactories(uint32_t count) {
   static const VideoPacketSenderFactory factories[] = {
     VideoPacketSenderFactory(30.00f, 150, 0x1234, 0.13f),
@@ -71,23 +39,6 @@ std::vector<const PacketSenderFactory*> VideoSenderFactories(uint32_t count) {
   return result;
 }
 
-std::vector<const PacketSenderFactory*> AdaptiveVideoSenderFactories(
-    uint32_t count) {
-  static const VideoPacketSenderFactory factories[] = {
-    AdaptiveVideoPacketSenderFactory(30.00f, 150, 0x1234, 0.13f),
-    AdaptiveVideoPacketSenderFactory(30.00f, 300, 0x3456, 0.26f),
-    AdaptiveVideoPacketSenderFactory(15.00f, 600, 0x4567, 0.39f),
-  };
-
-  assert(count <= sizeof(factories) / sizeof(factories[0]));
-
-  std::vector<const PacketSenderFactory*> result;
-  for (uint32_t i = 0; i < count; ++i) {
-    result.push_back(&factories[i]);
-  }
-  return result;
-}
-
 std::vector<BweTestConfig::EstimatorConfig> EstimatorConfigs() {
   static const RemoteBitrateEstimatorFactory factories[] = {
     RemoteBitrateEstimatorFactory(),
@@ -107,13 +58,6 @@ BweTestConfig MakeBweTestConfig(uint32_t sender_count) {
   return result;
 }
 
-BweTestConfig MakeAdaptiveBweTestConfig(uint32_t sender_count) {
-  BweTestConfig result = {
-    AdaptiveVideoSenderFactories(sender_count), EstimatorConfigs()
-  };
-  return result;
-}
-
 INSTANTIATE_TEST_CASE_P(VideoSendersTest, BweTest,
     ::testing::Values(MakeBweTestConfig(1),
                       MakeBweTestConfig(3)));
@@ -265,34 +209,6 @@ TEST_P(BweTest, Multi2) {
   jitter.SetJitter(120);
   RunFor(5 * 60 * 1000);
 }
-
-// This test fixture is used to instantiate tests running with adaptive video
-// senders.
-class AdaptiveBweTest : public BweTest {
- public:
-  AdaptiveBweTest() : BweTest() {}
-  virtual ~AdaptiveBweTest() {}
-
- private:
-
-  DISALLOW_COPY_AND_ASSIGN(AdaptiveBweTest);
-};
-
-INSTANTIATE_TEST_CASE_P(VideoSendersTest, AdaptiveBweTest,
-    ::testing::Values(MakeAdaptiveBweTestConfig(1),
-                      MakeAdaptiveBweTestConfig(3)));
-
-TEST_P(AdaptiveBweTest, SprintUplinkTest) {
-  TraceBasedDeliveryFilter filter(this);
-  ASSERT_TRUE(filter.Init(test::ResourcePath("sprint-uplink", "rx")));
-  RunFor(60 * 1000);
-}
-
-TEST_P(AdaptiveBweTest, Verizon4gDownlinkTest) {
-  TraceBasedDeliveryFilter filter(this);
-  ASSERT_TRUE(filter.Init(test::ResourcePath("verizon4g-downlink", "rx")));
-  RunFor(22 * 60 * 1000);
-}
 }  // namespace bwe
 }  // namespace testing
 }  // namespace webrtc

webrtc/modules/remote_bitrate_estimator/test/bwe_plot.sh

@@ -13,17 +13,29 @@
 #
 # In Eclipse, that amounts to creating a Run Configuration which starts
 # "/bin/bash" with the arguments "-c [trunk_path]/out/Debug/modules_unittests
-# --gtest_filter=*Estimators* | [trunk_path]/webrtc/modules/
-# remote_bitrate_estimator/bwe_plot.sh"
+# --gtest_filter=*BweTest* | [trunk_path]/webrtc/modules/
+# remote_bitrate_estimator/bwe_plot.
+
+# bwe_plot.sh has a single y axis and a dual y axis mode. If any line specifies
+# a an axis by ending with "#<axis number (1 or 2)>" two y axis will be used,
+# the first will be assumed to represent bitrate (in kbps) and the second will
+# be assumed to represent time deltas (in ms).
 
 log=$(</dev/stdin)
 
 function gen_gnuplot_input {
   colors=(a7001f 0a60c2 b2582b 21a66c d6604d 4393c3 f4a582 92c5de edcbb7 b1c5d0)
   data_sets=$(echo "$log" | grep "^PLOT" | cut -f 2 | sort | uniq)
+  linetypes=($(echo "$data_sets" | cut -d '#' -f 2 | cut -d ' ' -f 1))
   echo -n "reset; "
   echo -n "set terminal wxt size 1440,900 font \"Arial,9\"; "
   echo -n "set xtics 60; set xlabel \"Seconds\"; "
+  if [ -n $linetypes ]; then
+    echo -n "set ylabel 'bitrate (kbps)';"
+    echo -n "set ytics nomirror;"
+    echo -n "set y2label 'time delta (ms)';"
+    echo -n "set y2tics nomirror;"
+  fi
   echo -n "plot "
   i=0
   for set in $data_sets ; do
@@ -32,6 +44,13 @@ function gen_gnuplot_input {
     echo -n "linespoints "
     echo -n "ps 0.5 "
     echo -n "lc rgbcolor \"#${colors[$(($i % 10))]}\" "
+    if [ -n ${linetypes[$i - 1]} ]; then
+      echo -n "axes x1y${linetypes[$i - 1]} "
+    elif [ -n $linestypes ]; then
+      # If no line type is specified, but line types are used, we will default
+      # to the bitrate axis.
+      echo -n "axes x1y1 "
+    fi
     echo -n "title \"$set\" "
   done
   echo

webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc

@@ -69,6 +69,9 @@ class BweTest::TestedEstimator : public RemoteBitrateObserver {
     // time once packet reaches the estimator.
     int64_t packet_time_ms = (packet.send_time_us() + 500) / 1000;
     BWE_TEST_LOGGING_TIME(packet_time_ms);
+    BWE_TEST_LOGGING_PLOT("Delay_#2", clock_.TimeInMilliseconds(),
+                          packet_time_ms -
+                          (packet.creation_time_us() + 500) / 1000);
 
     int64_t step_ms = estimator_->TimeUntilNextProcess();
     while ((clock_.TimeInMilliseconds() + step_ms) < packet_time_ms) {
@@ -76,7 +79,6 @@ class BweTest::TestedEstimator : public RemoteBitrateObserver {
       estimator_->Process();
       step_ms = estimator_->TimeUntilNextProcess();
     }
-
     estimator_->IncomingPacket(packet_time_ms, packet.payload_size(),
                                packet.header());
     clock_.AdvanceTimeMilliseconds(packet_time_ms -
@@ -94,8 +96,8 @@ class BweTest::TestedEstimator : public RemoteBitrateObserver {
 
       double estimated_kbps = static_cast<double>(estimated_bps) / 1000.0;
       stats_.Push(estimated_kbps);
-      BWE_TEST_LOGGING_PLOT("Estimate", clock_.TimeInMilliseconds(),
-                            estimated_kbps / 1000.0);
+      BWE_TEST_LOGGING_PLOT("Estimate_#1", clock_.TimeInMilliseconds(),
+                            estimated_kbps);
       uint32_t relative_estimate_bps = 0;
       if (relative_estimator_ &&
           relative_estimator_->LatestEstimate(&relative_estimate_bps)) {
@@ -244,6 +246,7 @@ void BweTest::RunFor(int64_t time_ms) {
     for (vector<PacketProcessor*>::const_iterator it =
          processors_.begin(); it != processors_.end(); ++it) {
       (*it)->RunFor(simulation_interval_ms_, &packets);
+      (*it)->Plot((packets.back().send_time_us() + 500) / 1000);
     }
 
     // Verify packets are in order between batches.

webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.cc

@@ -45,20 +45,23 @@ int Random::Gaussian(int mean, int standard_deviation) {
 }
 
 Packet::Packet()
-    : send_time_us_(0),
+    : creation_time_us_(-1),
+      send_time_us_(-1),
       payload_size_(0) {
    memset(&header_, 0, sizeof(header_));
 }
 
 Packet::Packet(int64_t send_time_us, uint32_t payload_size,
-          const RTPHeader& header)
-  : send_time_us_(send_time_us),
+               const RTPHeader& header)
+  : creation_time_us_(send_time_us),
+    send_time_us_(send_time_us),
     payload_size_(payload_size),
     header_(header) {
 }
 
 Packet::Packet(int64_t send_time_us, uint32_t sequence_number)
-    : send_time_us_(send_time_us),
+    : creation_time_us_(send_time_us),
+      send_time_us_(send_time_us),
       payload_size_(0) {
    memset(&header_, 0, sizeof(header_));
    header_.sequenceNumber = sequence_number;
@@ -105,7 +108,21 @@ RateCounterFilter::RateCounterFilter(PacketProcessorListener* listener)
       last_accumulated_us_(0),
       window_(),
       pps_stats_(),
-      kbps_stats_() {
+      kbps_stats_(),
+      name_("") {
+}
+
+RateCounterFilter::RateCounterFilter(PacketProcessorListener* listener,
+                                     const std::string& name)
+    : PacketProcessor(listener),
+      kWindowSizeUs(1000000),
+      packets_per_second_(0),
+      bytes_per_second_(0),
+      last_accumulated_us_(0),
+      window_(),
+      pps_stats_(),
+      kbps_stats_(),
+      name_(name) {
 }
 
 RateCounterFilter::~RateCounterFilter() {
@@ -118,6 +135,12 @@ void RateCounterFilter::LogStats() {
   kbps_stats_.Log("kbps");
 }
 
+void RateCounterFilter::Plot(int64_t timestamp_ms) {
+  BWE_TEST_LOGGING_CONTEXT(name_.c_str());
+  BWE_TEST_LOGGING_PLOT("Throughput_#1", timestamp_ms,
+                        (bytes_per_second_ * 8) / 1000.0);
+}
+
 void RateCounterFilter::RunFor(int64_t /*time_ms*/, Packets* in_out) {
   assert(in_out);
   for (PacketsConstIt it = in_out->begin(); it != in_out->end(); ++it) {

webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h

@@ -135,17 +135,19 @@ class Packet {
  public:
   Packet();
   Packet(int64_t send_time_us, uint32_t payload_size,
-            const RTPHeader& header);
+         const RTPHeader& header);
   Packet(int64_t send_time_us, uint32_t sequence_number);
 
   bool operator<(const Packet& rhs) const;
 
+  int64_t creation_time_us() const { return creation_time_us_; }
   void set_send_time_us(int64_t send_time_us);
   int64_t send_time_us() const { return send_time_us_; }
   uint32_t payload_size() const { return payload_size_; }
   const RTPHeader& header() const { return header_; }
 
  private:
+  int64_t creation_time_us_;  // Time when the packet was created.
   int64_t send_time_us_;   // Time the packet left last processor touching it.
   uint32_t payload_size_;  // Size of the (non-existent, simulated) payload.
   RTPHeader header_;       // Actual contents.
@@ -172,6 +174,10 @@ class PacketProcessor {
   explicit PacketProcessor(PacketProcessorListener* listener);
   virtual ~PacketProcessor();
 
+  // Called after each simulation batch to allow the processor to plot any
+  // internal data.
+  virtual void Plot(int64_t timestamp_ms) {}
+
   // Run simulation for |time_ms| micro seconds, consuming packets from, and
   // producing packets into in_out. The outgoing packet list must be sorted on
   // |send_time_us_|. The simulation time |time_ms| is optional to use.
@@ -186,12 +192,15 @@ class PacketProcessor {
 class RateCounterFilter : public PacketProcessor {
  public:
   explicit RateCounterFilter(PacketProcessorListener* listener);
+  RateCounterFilter(PacketProcessorListener* listener,
+                    const std::string& context);
   virtual ~RateCounterFilter();
 
   uint32_t packets_per_second() const { return packets_per_second_; }
   uint32_t bits_per_second() const { return bytes_per_second_ * 8; }
 
   void LogStats();
+  virtual void Plot(int64_t timestamp_ms);
   virtual void RunFor(int64_t time_ms, Packets* in_out);
 
  private:
@@ -202,6 +211,7 @@ class RateCounterFilter : public PacketProcessor {
   Packets window_;
   Stats<double> pps_stats_;
   Stats<double> kbps_stats_;
+  std::string name_;
 
   DISALLOW_IMPLICIT_CONSTRUCTORS(RateCounterFilter);
 };
@@ -370,12 +380,44 @@ class AdaptiveVideoSender : public VideoSender {
                       uint32_t kbps, uint32_t ssrc, float first_frame_offset);
   virtual ~AdaptiveVideoSender() {}
 
-  virtual int64_t GetFeedbackIntervalMs() const { return 500; }
+  virtual int64_t GetFeedbackIntervalMs() const { return 100; }
   virtual void GiveFeedback(const Feedback& feedback);
 
 private:
   DISALLOW_IMPLICIT_CONSTRUCTORS(AdaptiveVideoSender);
 };
+
+class VideoPacketSenderFactory : public PacketSenderFactory {
+ public:
+  VideoPacketSenderFactory(float fps, uint32_t kbps, uint32_t ssrc,
+                           float frame_offset)
+      : fps_(fps),
+        kbps_(kbps),
+        ssrc_(ssrc),
+        frame_offset_(frame_offset) {
+  }
+  virtual ~VideoPacketSenderFactory() {}
+  virtual PacketSender* Create() const {
+    return new VideoSender(NULL, fps_, kbps_, ssrc_, frame_offset_);
+  }
+ protected:
+  float fps_;
+  uint32_t kbps_;
+  uint32_t ssrc_;
+  float frame_offset_;
+};
+
+class AdaptiveVideoPacketSenderFactory : public VideoPacketSenderFactory {
+ public:
+  AdaptiveVideoPacketSenderFactory(float fps, uint32_t kbps, uint32_t ssrc,
+                                   float frame_offset)
+      : VideoPacketSenderFactory(fps, kbps, ssrc, frame_offset) {}
+  virtual ~AdaptiveVideoPacketSenderFactory() {}
+  virtual PacketSender* Create() const {
+    return new AdaptiveVideoSender(NULL, fps_, kbps_, ssrc_, frame_offset_);
+  }
+};
+
 }  // namespace bwe
 }  // namespace testing
 }  // namespace webrtc