Adding BweFeedbackTest which tracks BWE performance over a set of simulated scenarios.

R=pbos@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/11019006

git-svn-id: http://webrtc.googlecode.com/svn/trunk@6015 4adac7df-926f-26a2-2b94-8c16560cd09d

webrtc/modules/remote_bitrate_estimator/bwe_simulations.cc

@@ -28,7 +28,7 @@ BweTestConfig::EstimatorConfig CreateEstimatorConfig(
                                         plot_delay, plot_estimate);
 }
 
-BweTestConfig MakeAdaptiveBweTestConfig(uint32_t sender_count) {
+BweTestConfig MakeAdaptiveBweTestConfig() {
   BweTestConfig result;
   result.estimator_configs.push_back(CreateEstimatorConfig(0, true, true));
   return result;
@@ -60,8 +60,7 @@ class BweSimulation : public BweTest,
 };
 
 INSTANTIATE_TEST_CASE_P(VideoSendersTest, BweSimulation,
-    ::testing::Values(MakeAdaptiveBweTestConfig(1),
+    ::testing::Values(MakeAdaptiveBweTestConfig()));
-                      MakeAdaptiveBweTestConfig(3)));
 
 TEST_P(BweSimulation, SprintUplinkTest) {
   VerboseLogging(true);

webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc

@@ -8,9 +8,12 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
+#include <sstream>
+
 #include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h"
 #include "webrtc/modules/remote_bitrate_estimator/test/bwe_test.h"
 #include "webrtc/test/testsupport/fileutils.h"
+#include "webrtc/test/testsupport/perf_test.h"
 
 using std::string;
 
@@ -27,16 +30,14 @@ BweTestConfig::EstimatorConfig EstimatorConfigs(Estimator estimator,
   };
   switch (estimator) {
     case kTransmissionOffset:
-      return BweTestConfig::EstimatorConfig("TOF", flow_id, &factories[0], true,
+      return BweTestConfig::EstimatorConfig("TOF", flow_id, &factories[0],
-                                            true);
+                                            kMimdControl, false, false);
     case kAbsSendTime:
-      return BweTestConfig::EstimatorConfig("AST", flow_id, &factories[1], true,
+      return BweTestConfig::EstimatorConfig("AST", flow_id, &factories[1],
-                                            true);
+                                            kMimdControl, false, false);
-    default:
-      assert(false);
-      return BweTestConfig::EstimatorConfig("AST", flow_id, &factories[1], true,
-                                            true);
   }
+  assert(false);
+  return BweTestConfig::EstimatorConfig();
 }
 
 struct DefaultBweTestConfig {
@@ -231,6 +232,107 @@ TEST_P(DefaultBweTest, Multi2) {
   jitter.SetJitter(120);
   RunFor(5 * 60 * 1000);
 }
+
+// This test fixture is used to instantiate tests running with adaptive video
+// senders.
+class BweFeedbackTest : public BweTest,
+                        public ::testing::TestWithParam<BweTestConfig> {
+ public:
+  BweFeedbackTest() : BweTest() {}
+  virtual ~BweFeedbackTest() {}
+
+  virtual void SetUp() {
+    BweTestConfig config;
+    config.estimator_configs.push_back(EstimatorConfigs(kAbsSendTime, 0));
+    SetupTestFromConfig(config);
+  }
+
+  void PrintResults(double max_throughput_kbps, Stats<double> throughput_kbps,
+                    Stats<double> delay_ms) {
+    double utilization = throughput_kbps.GetMean() / max_throughput_kbps;
+    webrtc::test::PrintResult("BwePerformance",
+                              GetTestName(),
+                              "Utilization",
+                              utilization * 100.0,
+                              "%",
+                              false);
+    std::stringstream ss;
+    ss << throughput_kbps.GetStdDev() / throughput_kbps.GetMean();
+    webrtc::test::PrintResult("BwePerformance",
+                              GetTestName(),
+                              "Utilization var coeff",
+                              ss.str(),
+                              "",
+                              false);
+    webrtc::test::PrintResult("BwePerformance",
+                              GetTestName(),
+                              "Average delay",
+                              delay_ms.AsString(),
+                              "ms",
+                              false);
+  }
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(BweFeedbackTest);
+};
+
+TEST_F(BweFeedbackTest, Choke1000kbps500kbps1000kbps) {
+  AdaptiveVideoSender sender(0, this, 30, 300, 0, 0);
+  ChokeFilter filter(this);
+  RateCounterFilter counter(this, "receiver_input");
+  const int kHighCapacityKbps = 1000;
+  const int kLowCapacityKbps = 500;
+  filter.SetCapacity(kHighCapacityKbps);
+  filter.SetMaxDelay(500);
+  RunFor(60 * 1000);
+  filter.SetCapacity(kLowCapacityKbps);
+  RunFor(60 * 1000);
+  filter.SetCapacity(kHighCapacityKbps);
+  RunFor(60 * 1000);
+  PrintResults((2 * kHighCapacityKbps + kLowCapacityKbps) / 3.0,
+               counter.GetBitrateStats(), filter.GetDelayStats());
+}
+
+TEST_F(BweFeedbackTest, Choke200kbps30kbps200kbps) {
+  AdaptiveVideoSender sender(0, this, 30, 300, 0, 0);
+  ChokeFilter filter(this);
+  RateCounterFilter counter(this, "receiver_input");
+  const int kHighCapacityKbps = 200;
+  const int kLowCapacityKbps = 30;
+  filter.SetCapacity(kHighCapacityKbps);
+  filter.SetMaxDelay(500);
+  RunFor(60 * 1000);
+  filter.SetCapacity(kLowCapacityKbps);
+  RunFor(60 * 1000);
+  filter.SetCapacity(kHighCapacityKbps);
+  RunFor(60 * 1000);
+
+  PrintResults((2 * kHighCapacityKbps + kLowCapacityKbps) / 3.0,
+               counter.GetBitrateStats(), filter.GetDelayStats());
+}
+
+TEST_F(BweFeedbackTest, Verizon4gDownlinkTest) {
+  AdaptiveVideoSender sender(0, this, 30, 300, 0, 0);
+  RateCounterFilter counter1(this, "sender_output");
+  TraceBasedDeliveryFilter filter(this, "link_capacity");
+  RateCounterFilter counter2(this, "receiver_input");
+  ASSERT_TRUE(filter.Init(test::ResourcePath("verizon4g-downlink", "rx")));
+  RunFor(22 * 60 * 1000);
+  PrintResults(filter.GetBitrateStats().GetMean(), counter2.GetBitrateStats(),
+               filter.GetDelayStats());
+}
+
+TEST_F(BweFeedbackTest, GoogleWifiTrace3Mbps) {
+  AdaptiveVideoSender sender(0, this, 30, 300, 0, 0);
+  RateCounterFilter counter1(this, "sender_output");
+  TraceBasedDeliveryFilter filter(this, "link_capacity");
+  filter.SetMaxDelay(500);
+  RateCounterFilter counter2(this, "receiver_input");
+  ASSERT_TRUE(filter.Init(test::ResourcePath("google-wifi-3mbps", "rx")));
+  RunFor(300 * 1000);
+  PrintResults(filter.GetBitrateStats().GetMean(), counter2.GetBitrateStats(),
+               filter.GetDelayStats());
+}
 }  // namespace bwe
 }  // namespace testing
 }  // namespace webrtc

webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc

@@ -342,6 +342,12 @@ void BweTest::RunFor(int64_t time_ms) {
     }
   }
 }
+
+string BweTest::GetTestName() const {
+  const ::testing::TestInfo* const test_info =
+      ::testing::UnitTest::GetInstance()->current_test_info();
+  return string(test_info->name());
+}
 }  // namespace bwe
 }  // namespace testing
 }  // namespace webrtc

webrtc/modules/remote_bitrate_estimator/test/bwe_test.h

@@ -101,6 +101,7 @@ class BweTest : public PacketProcessorListener {
   void SetupTestFromConfig(const BweTestConfig& config);
   void VerboseLogging(bool enable);
   void RunFor(int64_t time_ms);
+  std::string GetTestName() const;
 
  private:
   typedef std::map<int, TestedEstimator*> EstimatorMap;

webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.cc

@@ -191,7 +191,7 @@ PacketProcessor::~PacketProcessor() {
 RateCounterFilter::RateCounterFilter(PacketProcessorListener* listener)
     : PacketProcessor(listener, false),
       rate_counter_(new RateCounter()),
-      pps_stats_(),
+      packets_per_second_stats_(),
       kbps_stats_(),
       name_("") {}
 
@@ -199,7 +199,7 @@ RateCounterFilter::RateCounterFilter(PacketProcessorListener* listener,
                                      const std::string& name)
     : PacketProcessor(listener, false),
       rate_counter_(new RateCounter()),
-      pps_stats_(),
+      packets_per_second_stats_(),
       kbps_stats_(),
       name_(name) {}
 
@@ -208,7 +208,7 @@ RateCounterFilter::RateCounterFilter(PacketProcessorListener* listener,
                                      const std::string& name)
     : PacketProcessor(listener, flow_ids, false),
       rate_counter_(new RateCounter()),
-      pps_stats_(),
+      packets_per_second_stats_(),
       kbps_stats_(),
       name_(name) {
   std::stringstream ss;
@@ -233,7 +233,7 @@ uint32_t RateCounterFilter::bits_per_second() const {
 
 void RateCounterFilter::LogStats() {
   BWE_TEST_LOGGING_CONTEXT("RateCounterFilter");
-  pps_stats_.Log("pps");
+  packets_per_second_stats_.Log("pps");
   kbps_stats_.Log("kbps");
 }
 
@@ -252,7 +252,7 @@ void RateCounterFilter::RunFor(int64_t /*time_ms*/, Packets* in_out) {
   for (PacketsConstIt it = in_out->begin(); it != in_out->end(); ++it) {
     rate_counter_->UpdateRates(it->send_time_us(), it->payload_size());
   }
-  pps_stats_.Push(rate_counter_->packets_per_second());
+  packets_per_second_stats_.Push(rate_counter_->packets_per_second());
   kbps_stats_.Push(rate_counter_->bits_per_second() / 1000.0);
 }
 
@@ -418,7 +418,9 @@ TraceBasedDeliveryFilter::TraceBasedDeliveryFilter(
       local_time_us_(-1),
       rate_counter_(new RateCounter),
       name_(""),
-      delay_cap_helper_(new DelayCapHelper()) {}
+      delay_cap_helper_(new DelayCapHelper()),
+      packets_per_second_stats_(),
+      kbps_stats_() {}
 
 TraceBasedDeliveryFilter::TraceBasedDeliveryFilter(
     PacketProcessorListener* listener,
@@ -430,7 +432,9 @@ TraceBasedDeliveryFilter::TraceBasedDeliveryFilter(
       local_time_us_(-1),
       rate_counter_(new RateCounter),
       name_(name),
-      delay_cap_helper_(new DelayCapHelper()) {}
+      delay_cap_helper_(new DelayCapHelper()),
+      packets_per_second_stats_(),
+      kbps_stats_() {}
 
 TraceBasedDeliveryFilter::~TraceBasedDeliveryFilter() {
 }
@@ -491,6 +495,8 @@ void TraceBasedDeliveryFilter::RunFor(int64_t time_ms, Packets* in_out) {
     }
     ++it;
   }
+  packets_per_second_stats_.Push(rate_counter_->packets_per_second());
+  kbps_stats_.Push(rate_counter_->bits_per_second() / 1000.0);
 }
 
 void TraceBasedDeliveryFilter::SetMaxDelay(int max_delay_ms) {
@@ -501,6 +507,10 @@ Stats<double> TraceBasedDeliveryFilter::GetDelayStats() const {
   return delay_cap_helper_->delay_stats();
 }
 
+Stats<double> TraceBasedDeliveryFilter::GetBitrateStats() const {
+  return kbps_stats_;
+}
+
 void TraceBasedDeliveryFilter::ProceedToNextSlot() {
   if (*next_delivery_it_ <= local_time_us_) {
     ++next_delivery_it_;

webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h

@@ -17,6 +17,7 @@
 #include <algorithm>
 #include <list>
 #include <numeric>
+#include <sstream>
 #include <string>
 #include <vector>
 
@@ -88,6 +89,13 @@ template<typename T> class Stats {
     return max_;
   }
 
+  std::string AsString() {
+    std::stringstream ss;
+    ss << (GetMean() >= 0 ? GetMean() : -1) << ", " <<
+        (GetStdDev() >= 0 ? GetStdDev() : -1);
+    return ss.str();
+  }
+
   void Log(const std::string& units) {
     BWE_TEST_LOGGING_LOG5("", "%f %s\t+/-%f\t[%f,%f]",
         GetMean(), units.c_str(), GetStdDev(), GetMin(), GetMax());
@@ -226,7 +234,7 @@ class RateCounterFilter : public PacketProcessor {
 
  private:
   scoped_ptr<RateCounter> rate_counter_;
-  Stats<double> pps_stats_;
+  Stats<double> packets_per_second_stats_;
   Stats<double> kbps_stats_;
   std::string name_;
 
@@ -331,6 +339,7 @@ class TraceBasedDeliveryFilter : public PacketProcessor {
 
   void SetMaxDelay(int max_delay_ms);
   Stats<double> GetDelayStats() const;
+  Stats<double> GetBitrateStats() const;
 
  private:
   void ProceedToNextSlot();
@@ -343,6 +352,8 @@ class TraceBasedDeliveryFilter : public PacketProcessor {
   scoped_ptr<RateCounter> rate_counter_;
   std::string name_;
   scoped_ptr<DelayCapHelper> delay_cap_helper_;
+  Stats<double> packets_per_second_stats_;
+  Stats<double> kbps_stats_;
 
   DISALLOW_COPY_AND_ASSIGN(TraceBasedDeliveryFilter);
 };