diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_AST.bin.sha1 new file mode 100644 index 000000000..0e1c108af --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_AST.bin.sha1 @@ -0,0 +1 @@ +27bca982f40243ce6a491c05792f271166776eb9 \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_TOF.bin.sha1 new file mode 100644 index 000000000..0e1c108af --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_TOF.bin.sha1 @@ -0,0 +1 @@ +27bca982f40243ce6a491c05792f271166776eb9 \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_AST.bin.sha1 new file mode 100644 index 000000000..781691f3a --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_AST.bin.sha1 @@ -0,0 +1 @@ +2437de4b814694bfae0c7d47f851e356d8302796 \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_TOF.bin.sha1 new file mode 100644 index 000000000..781691f3a --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_TOF.bin.sha1 @@ -0,0 +1 @@ +2437de4b814694bfae0c7d47f851e356d8302796 \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_AST.bin.sha1 new file mode 100644 index 000000000..9a185074d --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_AST.bin.sha1 @@ -0,0 +1 @@ +c2d507d2d3edadab06942736a81c6ca3221e23c7 \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_TOF.bin.sha1 new file mode 100644 index 000000000..9a185074d --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_TOF.bin.sha1 @@ -0,0 +1 @@ +c2d507d2d3edadab06942736a81c6ca3221e23c7 \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_AST.bin.sha1 new file mode 100644 index 000000000..46ee7df41 --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_AST.bin.sha1 @@ -0,0 +1 @@ +feeeed364dc66dea552b26271094ff11707a1e65 \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_TOF.bin.sha1 new file mode 100644 index 000000000..46ee7df41 --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_TOF.bin.sha1 @@ -0,0 +1 @@ +feeeed364dc66dea552b26271094ff11707a1e65 \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_AST.bin.sha1 new file mode 100644 index 000000000..21cf98cea --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_AST.bin.sha1 @@ -0,0 +1 @@ +333ad9b7310be8d9c6a36c2b1fa63999a1098b88 \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_TOF.bin.sha1 new file mode 100644 index 000000000..21cf98cea --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_TOF.bin.sha1 @@ -0,0 +1 @@ +333ad9b7310be8d9c6a36c2b1fa63999a1098b88 \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_AST.bin.sha1 new file mode 100644 index 000000000..2fff90821 --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_AST.bin.sha1 @@ -0,0 +1 @@ +95a356db2eb39fd1bbf1ece1cfbf673b1f893487 \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_TOF.bin.sha1 new file mode 100644 index 000000000..2fff90821 --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_TOF.bin.sha1 @@ -0,0 +1 @@ +95a356db2eb39fd1bbf1ece1cfbf673b1f893487 \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_AST.bin.sha1 new file mode 100644 index 000000000..8e70aa2b2 --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_AST.bin.sha1 @@ -0,0 +1 @@ +d7814fe1106ee4d030bb5c4316765615f4b747dd \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_TOF.bin.sha1 new file mode 100644 index 000000000..8e70aa2b2 --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_TOF.bin.sha1 @@ -0,0 +1 @@ +d7814fe1106ee4d030bb5c4316765615f4b747dd \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_AST.bin.sha1 new file mode 100644 index 000000000..473aaa65a --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_AST.bin.sha1 @@ -0,0 +1 @@ +dba0be49e995fec988bb92d8d62b9e39b3f01bbe \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_TOF.bin.sha1 new file mode 100644 index 000000000..473aaa65a --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_TOF.bin.sha1 @@ -0,0 +1 @@ +dba0be49e995fec988bb92d8d62b9e39b3f01bbe \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_AST.bin.sha1 new file mode 100644 index 000000000..5d617bb9a --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_AST.bin.sha1 @@ -0,0 +1 @@ +091869a91d200add958904c0b5a42d47b1f5f9ac \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_TOF.bin.sha1 new file mode 100644 index 000000000..5d617bb9a --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_TOF.bin.sha1 @@ -0,0 +1 @@ +091869a91d200add958904c0b5a42d47b1f5f9ac \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_AST.bin.sha1 new file mode 100644 index 000000000..fcc16b6ce --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_AST.bin.sha1 @@ -0,0 +1 @@ +988a157505bb3b938c813282d90cb445a5cea19b \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_TOF.bin.sha1 new file mode 100644 index 000000000..fcc16b6ce --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_TOF.bin.sha1 @@ -0,0 +1 @@ +988a157505bb3b938c813282d90cb445a5cea19b \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_AST.bin.sha1 new file mode 100644 index 000000000..8395bcf06 --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_AST.bin.sha1 @@ -0,0 +1 @@ +e367a4e6b3ba1b4cd6ac2e71b55cdaed3b63a739 \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_TOF.bin.sha1 new file mode 100644 index 000000000..8395bcf06 --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_TOF.bin.sha1 @@ -0,0 +1 @@ +e367a4e6b3ba1b4cd6ac2e71b55cdaed3b63a739 \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_AST.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_AST.bin.sha1 new file mode 100644 index 000000000..df83598dc --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_AST.bin.sha1 @@ -0,0 +1 @@ +26e5e14a114a869a0777458520e6c27b5791863d \ No newline at end of file diff --git a/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_TOF.bin.sha1 b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_TOF.bin.sha1 new file mode 100644 index 000000000..df83598dc --- /dev/null +++ b/resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_TOF.bin.sha1 @@ -0,0 +1 @@ +26e5e14a114a869a0777458520e6c27b5791863d \ No newline at end of file diff --git a/webrtc/modules/modules.gyp b/webrtc/modules/modules.gyp index 579ae149f..53e7b316f 100644 --- a/webrtc/modules/modules.gyp +++ b/webrtc/modules/modules.gyp @@ -179,6 +179,10 @@ 'remote_bitrate_estimator/remote_bitrate_estimator_unittest_helper.h', 'remote_bitrate_estimator/remote_bitrate_estimators_test.cc', 'remote_bitrate_estimator/rtp_to_ntp_unittest.cc', + 'remote_bitrate_estimator/test/bwe_test_baselinefile.cc', + 'remote_bitrate_estimator/test/bwe_test_baselinefile.h', + 'remote_bitrate_estimator/test/bwe_test_fileutils.cc', + 'remote_bitrate_estimator/test/bwe_test_fileutils.h', 'remote_bitrate_estimator/test/bwe_test_framework.cc', 'remote_bitrate_estimator/test/bwe_test_framework.h', 'remote_bitrate_estimator/test/bwe_test_framework_unittest.cc', diff --git a/webrtc/modules/modules_unittests.isolate b/webrtc/modules/modules_unittests.isolate index ae72e75ed..32fc66287 100644 --- a/webrtc/modules/modules_unittests.isolate +++ b/webrtc/modules/modules_unittests.isolate @@ -46,6 +46,30 @@ '../../resources/near16_stereo.pcm', '../../resources/near32_stereo.pcm', '../../resources/near8_stereo.pcm', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_AST.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_0_TOF.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_AST.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke1_1_TOF.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_AST.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_0_TOF.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_AST.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingChoke2_1_TOF.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_AST.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingDelay1_0_TOF.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_AST.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_IncreasingLoss1_0_TOF.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_AST.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_Multi1_1_TOF.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_AST.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_0_TOF.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_AST.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyChoke_1_TOF.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_AST.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyDelay_0_TOF.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_AST.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_SteadyLoss_0_TOF.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_AST.bin', + '../../resources/remote_bitrate_estimator/VideoSendersTest_BweTest_UnlimitedSpeed_0_TOF.bin', '../../resources/video_coding/frame-ethernet-ii.pcap', '../../resources/video_coding/frame-loopback.pcap', '../../resources/video_coding/pltype103.rtp', diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc index 3046a5bdf..e04328cac 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc @@ -10,6 +10,7 @@ #include "webrtc/modules/remote_bitrate_estimator/test/bwe_test.h" +#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.h" #include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h" #include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" #include "webrtc/system_wrappers/interface/clock.h" @@ -35,15 +36,19 @@ template void DeleteElements(T* container) { class BweTest::TestedEstimator : public RemoteBitrateObserver { public: - explicit TestedEstimator(const BweTestConfig::EstimatorConfig& config) + TestedEstimator(const string& test_name, + const BweTestConfig::EstimatorConfig& config) : debug_name_(config.debug_name), clock_(0), stats_(), relative_estimator_stats_(), - latest_estimate_kbps_(-1.0), + latest_estimate_bps_(-1), estimator_(config.estimator_factory->Create(this, &clock_)), - relative_estimator_(NULL) { + relative_estimator_(NULL), + baseline_(BaseLineFileInterface::Create(test_name + "_" + debug_name_, + config.update_baseline)) { assert(estimator_.get()); + assert(baseline_.get()); // Default RTT in RemoteRateControl is 200 ms ; 50 ms is more realistic. estimator_->OnRttUpdate(50); } @@ -55,7 +60,7 @@ class BweTest::TestedEstimator : public RemoteBitrateObserver { void EatPacket(const Packet& packet) { BWE_TEST_LOGGING_CONTEXT(debug_name_); - latest_estimate_kbps_ = -1.0; + latest_estimate_bps_ = -1; // We're treating the send time (from previous filter) as the arrival // time once packet reaches the estimator. @@ -79,17 +84,22 @@ class BweTest::TestedEstimator : public RemoteBitrateObserver { bool CheckEstimate(PacketSender::Feedback* feedback) { assert(feedback); BWE_TEST_LOGGING_CONTEXT(debug_name_); - double estimated_kbps = 0.0; - if (LatestEstimate(&estimated_kbps)) { + uint32_t estimated_bps = 0; + if (LatestEstimate(&estimated_bps)) { + feedback->estimated_bps = estimated_bps; + baseline_->Estimate(clock_.TimeInMilliseconds(), estimated_bps); + + double estimated_kbps = static_cast(estimated_bps) / 1000.0; stats_.Push(estimated_kbps); BWE_TEST_LOGGING_PLOT("Estimate", clock_.TimeInMilliseconds(), - estimated_kbps / 100); - double relative_estimate_kbps = 0.0; + estimated_kbps / 1000.0); + uint32_t relative_estimate_bps = 0; if (relative_estimator_ && - relative_estimator_->LatestEstimate(&relative_estimate_kbps)) { + relative_estimator_->LatestEstimate(&relative_estimate_bps)) { + double relative_estimate_kbps = + static_cast(relative_estimate_bps) / 1000.0; relative_estimator_stats_.Push(estimated_kbps - relative_estimate_kbps); } - feedback->estimated_kbps = estimated_kbps; return true; } return false; @@ -105,32 +115,36 @@ class BweTest::TestedEstimator : public RemoteBitrateObserver { } } + void VerifyOrWriteBaseline() { + EXPECT_TRUE(baseline_->VerifyOrWrite()); + } + virtual void OnReceiveBitrateChanged(const vector& ssrcs, unsigned int bitrate) { } private: - bool LatestEstimate(double* estimate_kbps) { - if (latest_estimate_kbps_ < 0.0) { + bool LatestEstimate(uint32_t* estimate_bps) { + if (latest_estimate_bps_ < 0) { vector ssrcs; unsigned int bps = 0; if (!estimator_->LatestEstimate(&ssrcs, &bps)) { return false; } - latest_estimate_kbps_ = bps / 1000.0; + latest_estimate_bps_ = bps; } - *estimate_kbps = latest_estimate_kbps_; + *estimate_bps = latest_estimate_bps_; return true; } string debug_name_; - bool log_estimates_; SimulatedClock clock_; Stats stats_; Stats relative_estimator_stats_; - double latest_estimate_kbps_; + int64_t latest_estimate_bps_; scoped_ptr estimator_; TestedEstimator* relative_estimator_; + scoped_ptr baseline_; DISALLOW_IMPLICIT_CONSTRUCTORS(TestedEstimator); }; @@ -150,8 +164,11 @@ BweTest::~BweTest() { } void BweTest::SetUp() { - BWE_TEST_LOGGING_GLOBAL_CONTEXT(::testing::UnitTest::GetInstance()-> - current_test_info()->test_case_name()); + const ::testing::TestInfo* const test_info = + ::testing::UnitTest::GetInstance()->current_test_info(); + string test_name = + string(test_info->test_case_name()) + "_" + string(test_info->name()); + BWE_TEST_LOGGING_GLOBAL_CONTEXT(test_name); const BweTestConfig& config = GetParam(); @@ -175,7 +192,7 @@ void BweTest::SetUp() { for (vector:: const_iterator it = config.estimator_configs.begin(); it != config.estimator_configs.end(); ++it) { - estimators_.push_back(new TestedEstimator(*it)); + estimators_.push_back(new TestedEstimator(test_name, *it)); } if (estimators_.size() > 1) { // Set all estimators as relative to the first one. @@ -189,7 +206,13 @@ void BweTest::SetUp() { void BweTest::TearDown() { BWE_TEST_LOGGING_GLOBAL_ENABLE(true); - LogStats(); + + for (vector::iterator eit = estimators_.begin(); + eit != estimators_.end(); ++eit) { + (*eit)->VerifyOrWriteBaseline(); + (*eit)->LogStats(); + } + BWE_TEST_LOGGING_GLOBAL_CONTEXT(""); } @@ -251,13 +274,6 @@ void BweTest::RunFor(int64_t time_ms) { } } } - -void BweTest::LogStats() { - for (vector::iterator eit = estimators_.begin(); - eit != estimators_.end(); ++eit) { - (*eit)->LogStats(); - } -} } // namespace bwe } // namespace testing } // namespace webrtc diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.h b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.h index 0945ea6a1..286720b0c 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test.h +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test.h @@ -23,14 +23,27 @@ namespace bwe { struct BweTestConfig { struct EstimatorConfig { - EstimatorConfig() : debug_name(), estimator_factory(NULL) {} + EstimatorConfig() + : debug_name(), + estimator_factory(NULL), + update_baseline(false) { + } EstimatorConfig(std::string debug_name, const RemoteBitrateEstimatorFactory* estimator_factory) : debug_name(debug_name), - estimator_factory(estimator_factory) { + estimator_factory(estimator_factory), + update_baseline(false) { + } + EstimatorConfig(std::string debug_name, + const RemoteBitrateEstimatorFactory* estimator_factory, + bool update_baseline) + : debug_name(debug_name), + estimator_factory(estimator_factory), + update_baseline(update_baseline) { } std::string debug_name; const RemoteBitrateEstimatorFactory* estimator_factory; + bool update_baseline; }; std::vector sender_factories; @@ -51,7 +64,6 @@ class BweTest : public ::testing::TestWithParam, protected: void VerboseLogging(bool enable); void RunFor(int64_t time_ms); - void LogStats(); private: class TestedEstimator; diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.cc new file mode 100644 index 000000000..30723be99 --- /dev/null +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.cc @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.h" + +#include +#include +#include + +#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.h" +#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_logging.h" +#include "webrtc/system_wrappers/interface/constructor_magic.h" +#include "webrtc/system_wrappers/interface/scoped_ptr.h" +#include "webrtc/test/testsupport/fileutils.h" + +namespace webrtc { +namespace testing { +namespace bwe { + +// The format of BWE test baseline files is extremely simple: +// 1. All read/written entities are 32-bit unsigned integers in network byte +// order (Big Endian). +// 2. Files beging with a 2 word header containing a magic marker and file +// format version indicator. The Magic marker reads "BWE!" in a hex dump. +// 3. Each estimate is logged as a pair of words: time in milliseconds and +// estimated bit rate, in bits per second. +const uint32_t kMagicMarker = 0x42574521; +const uint32_t kFileVersion1 = 0x00000001; +const char kResourceSubDir[] = "remote_bitrate_estimator"; + +class BaseLineFileVerify : public BaseLineFileInterface { + public: + // If |allow_missing_file| is set, VerifyOrWrite() will return true even if + // the baseline file is missing. This is the default when verifying files, but + // not when updating (i.e. we always write it out if missing). + BaseLineFileVerify(const std::string& filepath, bool allow_missing_file) + : reader_(), + fail_to_read_response_(false) { + scoped_ptr reader; + reader.reset(ResourceFileReader::Create(filepath, "bin")); + if (!reader.get()) { + printf("WARNING: Missing baseline file for BWE test: %s.bin\n", + filepath.c_str()); + fail_to_read_response_ = allow_missing_file; + } else { + uint32_t magic_marker = 0; + uint32_t file_version = 0; + if (reader->Read(&magic_marker) && magic_marker == kMagicMarker && + reader->Read(&file_version) && file_version == kFileVersion1) { + reader_.swap(reader); + } else { + printf("WARNING: Bad baseline file header for BWE test: %s.bin\n", + filepath.c_str()); + } + } + } + virtual ~BaseLineFileVerify() {} + + virtual void Estimate(int64_t time_ms, uint32_t estimate_bps) { + if (reader_.get()) { + uint32_t read_ms = 0; + uint32_t read_bps = 0; + if (reader_->Read(&read_ms) && read_ms == time_ms && + reader_->Read(&read_bps) && read_bps == estimate_bps) { + } else { + printf("ERROR: Baseline differs starting at: %d ms (%d vs %d)!\n", + static_cast(time_ms), estimate_bps, read_bps); + reader_.reset(NULL); + } + } + } + + virtual bool VerifyOrWrite() { + if (reader_.get()) { + if (reader_->IsAtEnd()) { + return true; + } else { + printf("ERROR: Baseline file contains more data!\n"); + return false; + } + } + return fail_to_read_response_; + } + + private: + scoped_ptr reader_; + bool fail_to_read_response_; + + DISALLOW_IMPLICIT_CONSTRUCTORS(BaseLineFileVerify); +}; + +class BaseLineFileUpdate : public BaseLineFileInterface { + public: + BaseLineFileUpdate(const std::string& filepath, + BaseLineFileInterface* verifier) + : verifier_(verifier), + output_content_(), + filepath_(filepath) { + output_content_.push_back(kMagicMarker); + output_content_.push_back(kFileVersion1); + } + virtual ~BaseLineFileUpdate() {} + + virtual void Estimate(int64_t time_ms, uint32_t estimate_bps) { + verifier_->Estimate(time_ms, estimate_bps); + output_content_.push_back(static_cast(time_ms)); + output_content_.push_back(estimate_bps); + } + + virtual bool VerifyOrWrite() { + if (!verifier_->VerifyOrWrite()) { + std::string dir_path = webrtc::test::OutputPath() + kResourceSubDir; + if (!webrtc::test::CreateDirectory(dir_path)) { + printf("WARNING: Cannot create output dir: %s\n", dir_path.c_str()); + return false; + } + scoped_ptr writer; + writer.reset(OutputFileWriter::Create(filepath_, "bin")); + if (!writer.get()) { + printf("WARNING: Cannot create output file: %s.bin\n", + filepath_.c_str()); + return false; + } + printf("NOTE: Writing baseline file for BWE test: %s.bin\n", + filepath_.c_str()); + for (std::vector::iterator it = output_content_.begin(); + it != output_content_.end(); ++it) { + writer->Write(*it); + } + return true; + } + printf("NOTE: No change, not writing: %s\n", filepath_.c_str()); + return true; + } + + private: + scoped_ptr verifier_; + std::vector output_content_; + std::string filepath_; + + DISALLOW_IMPLICIT_CONSTRUCTORS(BaseLineFileUpdate); +}; + +BaseLineFileInterface* BaseLineFileInterface::Create( + const std::string& filename, bool write_output_file) { + std::string filepath = filename; + std::replace(filepath.begin(), filepath.end(), '/', '_'); + filepath = std::string(kResourceSubDir) + "/" + filepath; + + scoped_ptr result; + result.reset(new BaseLineFileVerify(filepath, !write_output_file)); + if (write_output_file) { + // Takes ownership of the |verifier| instance. + result.reset(new BaseLineFileUpdate(filepath, result.release())); + } + return result.release(); +} +} // namespace bwe +} // namespace testing +} // namespace webrtc diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.h b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.h new file mode 100644 index 000000000..64dfa8553 --- /dev/null +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_baselinefile.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_BASELINEFILE_H_ +#define WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_BASELINEFILE_H_ + +#include +#include "webrtc/modules/interface/module_common_types.h" + +namespace webrtc { +namespace testing { +namespace bwe { + +class BaseLineFileInterface { + public: + virtual ~BaseLineFileInterface() {} + + // Compare, or log, one estimate against the baseline file. + virtual void Estimate(int64_t time_ms, uint32_t estimate_bps) = 0; + + // Verify whether there are any differences between the logged estimates and + // those read from the baseline file. If updating the baseline file, write out + // new file if there were differences. Return true if logged estimates are + // identical, or if output file was updated successfully. + virtual bool VerifyOrWrite() = 0; + + // Create an instance for either verifying estimates against a baseline file + // with name |filename|, living in the resources/ directory or, if the flag + // |write_updated_file| is set, write logged estimates to a file with the same + // name, living in the out/ directory. + static BaseLineFileInterface* Create(const std::string& filename, + bool write_updated_file); +}; +} // namespace bwe +} // namespace testing +} // namespace webrtc + +#endif // WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_BASELINEFILE_H_ diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.cc new file mode 100644 index 000000000..4e5d4074a --- /dev/null +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.cc @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.h" + +#ifdef WIN32 +#include +#else +#include +#endif +#include + +#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_logging.h" +#include "webrtc/system_wrappers/interface/scoped_ptr.h" +#include "webrtc/test/testsupport/fileutils.h" + +namespace webrtc { +namespace testing { +namespace bwe { + +ResourceFileReader::~ResourceFileReader() { + if (file_ != NULL) { + fclose(file_); + file_ = NULL; + } +} + +bool ResourceFileReader::IsAtEnd() { + int32_t current_pos = ftell(file_); + fseek(file_, 0, SEEK_END); + int32_t end_pos = ftell(file_); + fseek(file_, current_pos, SEEK_SET); + return current_pos == end_pos; +} + +bool ResourceFileReader::Read(uint32_t* out) { + assert(out); + uint32_t tmp = 0; + if (fread(&tmp, 1, sizeof(uint32_t), file_) != sizeof(uint32_t)) { + printf("Error reading!\n"); + return false; + } + *out = ntohl(tmp); + return true; +} + +ResourceFileReader* ResourceFileReader::Create(const std::string& filename, + const std::string& extension) { + std::string filepath = webrtc::test::ResourcePath(filename, extension); + FILE* file = fopen(filepath.c_str(), "rb"); + if (file == NULL) { + BWE_TEST_LOGGING_CONTEXT("ResourceFileReader"); + BWE_TEST_LOGGING_LOG1("Create", "Can't read file: %s", filepath.c_str()); + return 0; + } else { + return new ResourceFileReader(file); + } +} + +OutputFileWriter::~OutputFileWriter() { + if (file_ != NULL) { + fclose(file_); + file_ = NULL; + } +} + +bool OutputFileWriter::Write(uint32_t value) { + uint32_t tmp = htonl(value); + if (fwrite(&tmp, 1, sizeof(uint32_t), file_) != sizeof(uint32_t)) { + return false; + } + return true; +} + +OutputFileWriter* OutputFileWriter::Create(const std::string& filename, + const std::string& extension) { + std::string filepath = webrtc::test::OutputPath() + filename + "." + + extension; + FILE* file = fopen(filepath.c_str(), "wb"); + if (file == NULL) { + BWE_TEST_LOGGING_CONTEXT("OutputFileWriter"); + BWE_TEST_LOGGING_LOG1("Create", "Can't write file: %s", filepath.c_str()); + return NULL; + } else { + return new OutputFileWriter(file); + } +} +} // namespace bwe +} // namespace testing +} // namespace webrtc diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.h b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.h new file mode 100644 index 000000000..397a16977 --- /dev/null +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_fileutils.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_FILEUTILS_H_ +#define WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_FILEUTILS_H_ + +#include +#include + +#include "webrtc/modules/interface/module_common_types.h" +#include "webrtc/system_wrappers/interface/constructor_magic.h" + +namespace webrtc { +namespace testing { +namespace bwe { + +class ResourceFileReader { + public: + ~ResourceFileReader(); + + bool IsAtEnd(); + bool Read(uint32_t* out); + + static ResourceFileReader* Create(const std::string& filename, + const std::string& extension); + + private: + explicit ResourceFileReader(FILE* file) : file_(file) {} + FILE* file_; + DISALLOW_IMPLICIT_CONSTRUCTORS(ResourceFileReader); +}; + +class OutputFileWriter { + public: + ~OutputFileWriter(); + + bool Write(uint32_t value); + + static OutputFileWriter* Create(const std::string& filename, + const std::string& extension); + + private: + explicit OutputFileWriter(FILE* file) : file_(file) {} + FILE* file_; + DISALLOW_IMPLICIT_CONSTRUCTORS(OutputFileWriter); +}; +} // namespace bwe +} // namespace testing +} // namespace webrtc + +#endif // WEBRTC_MODULES_REMOTE_BITRATE_ESTIMATOR_TEST_BWE_TEST_FILEUTILS_H_ diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h index c19cb2b80..71fa0ea6e 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h @@ -288,7 +288,7 @@ class ChokeFilter : public PacketProcessor { class PacketSender : public PacketProcessor { public: struct Feedback { - double estimated_kbps; + uint32_t estimated_bps; }; explicit PacketSender(PacketProcessorListener* listener);