diff --git a/webrtc/modules/modules.gyp b/webrtc/modules/modules.gyp index db3281fe2..d1c0bd1a7 100644 --- a/webrtc/modules/modules.gyp +++ b/webrtc/modules/modules.gyp @@ -175,6 +175,7 @@ 'media_file/source/media_file_unittest.cc', 'module_common_types_unittest.cc', 'pacing/paced_sender_unittest.cc', + 'remote_bitrate_estimator/bwe_simulations.cc', 'remote_bitrate_estimator/include/mock/mock_remote_bitrate_observer.h', 'remote_bitrate_estimator/rate_statistics_unittest.cc', 'remote_bitrate_estimator/remote_bitrate_estimator_single_stream_unittest.cc', diff --git a/webrtc/modules/remote_bitrate_estimator/bwe_simulations.cc b/webrtc/modules/remote_bitrate_estimator/bwe_simulations.cc new file mode 100644 index 000000000..797d669f8 --- /dev/null +++ b/webrtc/modules/remote_bitrate_estimator/bwe_simulations.cc @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" +#include "webrtc/modules/remote_bitrate_estimator/test/bwe_test.h" +#include "webrtc/test/testsupport/fileutils.h" + +namespace webrtc { +namespace testing { +namespace bwe { +#if BWE_TEST_LOGGING_COMPILE_TIME_ENABLE +std::vector SingleEstimatorConfig() { + static const RemoteBitrateEstimatorFactory factory = + AbsoluteSendTimeRemoteBitrateEstimatorFactory(); + + std::vector result; + result.push_back(BweTestConfig::EstimatorConfig("AST", &factory)); + return result; +} + +std::vector AdaptiveVideoSenderFactories( + uint32_t count) { + static const AdaptiveVideoPacketSenderFactory factories[] = { + AdaptiveVideoPacketSenderFactory(30.00f, 150, 0x1234, 0.13f), + AdaptiveVideoPacketSenderFactory(30.00f, 300, 0x3456, 0.26f), + AdaptiveVideoPacketSenderFactory(15.00f, 600, 0x4567, 0.39f), + }; + + assert(count <= sizeof(factories) / sizeof(factories[0])); + + std::vector result; + for (uint32_t i = 0; i < count; ++i) { + result.push_back(&factories[i]); + } + return result; +} + +BweTestConfig MakeAdaptiveBweTestConfig(uint32_t sender_count) { + BweTestConfig result = { + AdaptiveVideoSenderFactories(sender_count), SingleEstimatorConfig() + }; + return result; +} + +// This test fixture is used to instantiate tests running with adaptive video +// senders. +class BweSimulation : public BweTest { + public: + BweSimulation() : BweTest() {} + virtual ~BweSimulation() {} + + private: + DISALLOW_COPY_AND_ASSIGN(BweSimulation); +}; + +INSTANTIATE_TEST_CASE_P(VideoSendersTest, BweSimulation, + ::testing::Values(MakeAdaptiveBweTestConfig(1), + MakeAdaptiveBweTestConfig(3))); + +TEST_P(BweSimulation, SprintUplinkTest) { + VerboseLogging(true); + RateCounterFilter counter1(this, "sender_output"); + TraceBasedDeliveryFilter filter(this); + RateCounterFilter counter2(this, "receiver_input"); + ASSERT_TRUE(filter.Init(test::ResourcePath("sprint-uplink", "rx"))); + RunFor(60 * 1000); +} + +TEST_P(BweSimulation, Verizon4gDownlinkTest) { + VerboseLogging(true); + RateCounterFilter counter1(this, "sender_output"); + TraceBasedDeliveryFilter filter(this); + RateCounterFilter counter2(this, "receiver_input"); + ASSERT_TRUE(filter.Init(test::ResourcePath("verizon4g-downlink", "rx"))); + RunFor(22 * 60 * 1000); +} +#endif // BWE_TEST_LOGGING_COMPILE_TIME_ENABLE +} // namespace bwe +} // namespace testing +} // namespace webrtc diff --git a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc index 5b6ed7a88..3d86fbb1f 100644 --- a/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc +++ b/webrtc/modules/remote_bitrate_estimator/remote_bitrate_estimators_test.cc @@ -15,38 +15,6 @@ namespace webrtc { namespace testing { namespace bwe { - -class VideoPacketSenderFactory : public PacketSenderFactory { - public: - VideoPacketSenderFactory(float fps, uint32_t kbps, uint32_t ssrc, - float frame_offset) - : fps_(fps), - kbps_(kbps), - ssrc_(ssrc), - frame_offset_(frame_offset) { - } - virtual ~VideoPacketSenderFactory() {} - virtual PacketSender* Create() const { - return new VideoSender(NULL, fps_, kbps_, ssrc_, frame_offset_); - } - protected: - float fps_; - uint32_t kbps_; - uint32_t ssrc_; - float frame_offset_; - bool adaptive_; -}; - -class AdaptiveVideoPacketSenderFactory : public VideoPacketSenderFactory { - public: - AdaptiveVideoPacketSenderFactory(float fps, uint32_t kbps, uint32_t ssrc, - float frame_offset) - : VideoPacketSenderFactory(fps, kbps, ssrc, frame_offset) {} - virtual PacketSender* Create() const { - return new AdaptiveVideoSender(NULL, fps_, kbps_, ssrc_, frame_offset_); - } -}; - std::vector VideoSenderFactories(uint32_t count) { static const VideoPacketSenderFactory factories[] = { VideoPacketSenderFactory(30.00f, 150, 0x1234, 0.13f), @@ -71,23 +39,6 @@ std::vector VideoSenderFactories(uint32_t count) { return result; } -std::vector AdaptiveVideoSenderFactories( - uint32_t count) { - static const VideoPacketSenderFactory factories[] = { - AdaptiveVideoPacketSenderFactory(30.00f, 150, 0x1234, 0.13f), - AdaptiveVideoPacketSenderFactory(30.00f, 300, 0x3456, 0.26f), - AdaptiveVideoPacketSenderFactory(15.00f, 600, 0x4567, 0.39f), - }; - - assert(count <= sizeof(factories) / sizeof(factories[0])); - - std::vector result; - for (uint32_t i = 0; i < count; ++i) { - result.push_back(&factories[i]); - } - return result; -} - std::vector EstimatorConfigs() { static const RemoteBitrateEstimatorFactory factories[] = { RemoteBitrateEstimatorFactory(), @@ -107,13 +58,6 @@ BweTestConfig MakeBweTestConfig(uint32_t sender_count) { return result; } -BweTestConfig MakeAdaptiveBweTestConfig(uint32_t sender_count) { - BweTestConfig result = { - AdaptiveVideoSenderFactories(sender_count), EstimatorConfigs() - }; - return result; -} - INSTANTIATE_TEST_CASE_P(VideoSendersTest, BweTest, ::testing::Values(MakeBweTestConfig(1), MakeBweTestConfig(3))); @@ -265,34 +209,6 @@ TEST_P(BweTest, Multi2) { jitter.SetJitter(120); RunFor(5 * 60 * 1000); } - -// This test fixture is used to instantiate tests running with adaptive video -// senders. -class AdaptiveBweTest : public BweTest { - public: - AdaptiveBweTest() : BweTest() {} - virtual ~AdaptiveBweTest() {} - - private: - - DISALLOW_COPY_AND_ASSIGN(AdaptiveBweTest); -}; - -INSTANTIATE_TEST_CASE_P(VideoSendersTest, AdaptiveBweTest, - ::testing::Values(MakeAdaptiveBweTestConfig(1), - MakeAdaptiveBweTestConfig(3))); - -TEST_P(AdaptiveBweTest, SprintUplinkTest) { - TraceBasedDeliveryFilter filter(this); - ASSERT_TRUE(filter.Init(test::ResourcePath("sprint-uplink", "rx"))); - RunFor(60 * 1000); -} - -TEST_P(AdaptiveBweTest, Verizon4gDownlinkTest) { - TraceBasedDeliveryFilter filter(this); - ASSERT_TRUE(filter.Init(test::ResourcePath("verizon4g-downlink", "rx"))); - RunFor(22 * 60 * 1000); -} } // namespace bwe } // namespace testing } // namespace webrtc diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_plot.sh b/webrtc/modules/remote_bitrate_estimator/test/bwe_plot.sh index d7b1a032b..627b930b0 100755 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe_plot.sh +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_plot.sh @@ -13,17 +13,29 @@ # # In Eclipse, that amounts to creating a Run Configuration which starts # "/bin/bash" with the arguments "-c [trunk_path]/out/Debug/modules_unittests -# --gtest_filter=*Estimators* | [trunk_path]/webrtc/modules/ -# remote_bitrate_estimator/bwe_plot.sh" +# --gtest_filter=*BweTest* | [trunk_path]/webrtc/modules/ +# remote_bitrate_estimator/bwe_plot. + +# bwe_plot.sh has a single y axis and a dual y axis mode. If any line specifies +# a an axis by ending with "#" two y axis will be used, +# the first will be assumed to represent bitrate (in kbps) and the second will +# be assumed to represent time deltas (in ms). log=$(TimeUntilNextProcess(); while ((clock_.TimeInMilliseconds() + step_ms) < packet_time_ms) { @@ -76,7 +79,6 @@ class BweTest::TestedEstimator : public RemoteBitrateObserver { estimator_->Process(); step_ms = estimator_->TimeUntilNextProcess(); } - estimator_->IncomingPacket(packet_time_ms, packet.payload_size(), packet.header()); clock_.AdvanceTimeMilliseconds(packet_time_ms - @@ -94,8 +96,8 @@ class BweTest::TestedEstimator : public RemoteBitrateObserver { double estimated_kbps = static_cast(estimated_bps) / 1000.0; stats_.Push(estimated_kbps); - BWE_TEST_LOGGING_PLOT("Estimate", clock_.TimeInMilliseconds(), - estimated_kbps / 1000.0); + BWE_TEST_LOGGING_PLOT("Estimate_#1", clock_.TimeInMilliseconds(), + estimated_kbps); uint32_t relative_estimate_bps = 0; if (relative_estimator_ && relative_estimator_->LatestEstimate(&relative_estimate_bps)) { @@ -244,6 +246,7 @@ void BweTest::RunFor(int64_t time_ms) { for (vector::const_iterator it = processors_.begin(); it != processors_.end(); ++it) { (*it)->RunFor(simulation_interval_ms_, &packets); + (*it)->Plot((packets.back().send_time_us() + 500) / 1000); } // Verify packets are in order between batches. diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.cc b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.cc index 4a74f62c2..b4a6a09e7 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.cc +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.cc @@ -45,20 +45,23 @@ int Random::Gaussian(int mean, int standard_deviation) { } Packet::Packet() - : send_time_us_(0), + : creation_time_us_(-1), + send_time_us_(-1), payload_size_(0) { memset(&header_, 0, sizeof(header_)); } Packet::Packet(int64_t send_time_us, uint32_t payload_size, - const RTPHeader& header) - : send_time_us_(send_time_us), + const RTPHeader& header) + : creation_time_us_(send_time_us), + send_time_us_(send_time_us), payload_size_(payload_size), header_(header) { } Packet::Packet(int64_t send_time_us, uint32_t sequence_number) - : send_time_us_(send_time_us), + : creation_time_us_(send_time_us), + send_time_us_(send_time_us), payload_size_(0) { memset(&header_, 0, sizeof(header_)); header_.sequenceNumber = sequence_number; @@ -105,7 +108,21 @@ RateCounterFilter::RateCounterFilter(PacketProcessorListener* listener) last_accumulated_us_(0), window_(), pps_stats_(), - kbps_stats_() { + kbps_stats_(), + name_("") { +} + +RateCounterFilter::RateCounterFilter(PacketProcessorListener* listener, + const std::string& name) + : PacketProcessor(listener), + kWindowSizeUs(1000000), + packets_per_second_(0), + bytes_per_second_(0), + last_accumulated_us_(0), + window_(), + pps_stats_(), + kbps_stats_(), + name_(name) { } RateCounterFilter::~RateCounterFilter() { @@ -118,6 +135,12 @@ void RateCounterFilter::LogStats() { kbps_stats_.Log("kbps"); } +void RateCounterFilter::Plot(int64_t timestamp_ms) { + BWE_TEST_LOGGING_CONTEXT(name_.c_str()); + BWE_TEST_LOGGING_PLOT("Throughput_#1", timestamp_ms, + (bytes_per_second_ * 8) / 1000.0); +} + void RateCounterFilter::RunFor(int64_t /*time_ms*/, Packets* in_out) { assert(in_out); for (PacketsConstIt it = in_out->begin(); it != in_out->end(); ++it) { diff --git a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h index 9dd218aef..d7057da7d 100644 --- a/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h +++ b/webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h @@ -135,17 +135,19 @@ class Packet { public: Packet(); Packet(int64_t send_time_us, uint32_t payload_size, - const RTPHeader& header); + const RTPHeader& header); Packet(int64_t send_time_us, uint32_t sequence_number); bool operator<(const Packet& rhs) const; + int64_t creation_time_us() const { return creation_time_us_; } void set_send_time_us(int64_t send_time_us); int64_t send_time_us() const { return send_time_us_; } uint32_t payload_size() const { return payload_size_; } const RTPHeader& header() const { return header_; } private: + int64_t creation_time_us_; // Time when the packet was created. int64_t send_time_us_; // Time the packet left last processor touching it. uint32_t payload_size_; // Size of the (non-existent, simulated) payload. RTPHeader header_; // Actual contents. @@ -172,6 +174,10 @@ class PacketProcessor { explicit PacketProcessor(PacketProcessorListener* listener); virtual ~PacketProcessor(); + // Called after each simulation batch to allow the processor to plot any + // internal data. + virtual void Plot(int64_t timestamp_ms) {} + // Run simulation for |time_ms| micro seconds, consuming packets from, and // producing packets into in_out. The outgoing packet list must be sorted on // |send_time_us_|. The simulation time |time_ms| is optional to use. @@ -186,12 +192,15 @@ class PacketProcessor { class RateCounterFilter : public PacketProcessor { public: explicit RateCounterFilter(PacketProcessorListener* listener); + RateCounterFilter(PacketProcessorListener* listener, + const std::string& context); virtual ~RateCounterFilter(); uint32_t packets_per_second() const { return packets_per_second_; } uint32_t bits_per_second() const { return bytes_per_second_ * 8; } void LogStats(); + virtual void Plot(int64_t timestamp_ms); virtual void RunFor(int64_t time_ms, Packets* in_out); private: @@ -202,6 +211,7 @@ class RateCounterFilter : public PacketProcessor { Packets window_; Stats pps_stats_; Stats kbps_stats_; + std::string name_; DISALLOW_IMPLICIT_CONSTRUCTORS(RateCounterFilter); }; @@ -370,12 +380,44 @@ class AdaptiveVideoSender : public VideoSender { uint32_t kbps, uint32_t ssrc, float first_frame_offset); virtual ~AdaptiveVideoSender() {} - virtual int64_t GetFeedbackIntervalMs() const { return 500; } + virtual int64_t GetFeedbackIntervalMs() const { return 100; } virtual void GiveFeedback(const Feedback& feedback); private: DISALLOW_IMPLICIT_CONSTRUCTORS(AdaptiveVideoSender); }; + +class VideoPacketSenderFactory : public PacketSenderFactory { + public: + VideoPacketSenderFactory(float fps, uint32_t kbps, uint32_t ssrc, + float frame_offset) + : fps_(fps), + kbps_(kbps), + ssrc_(ssrc), + frame_offset_(frame_offset) { + } + virtual ~VideoPacketSenderFactory() {} + virtual PacketSender* Create() const { + return new VideoSender(NULL, fps_, kbps_, ssrc_, frame_offset_); + } + protected: + float fps_; + uint32_t kbps_; + uint32_t ssrc_; + float frame_offset_; +}; + +class AdaptiveVideoPacketSenderFactory : public VideoPacketSenderFactory { + public: + AdaptiveVideoPacketSenderFactory(float fps, uint32_t kbps, uint32_t ssrc, + float frame_offset) + : VideoPacketSenderFactory(fps, kbps, ssrc, frame_offset) {} + virtual ~AdaptiveVideoPacketSenderFactory() {} + virtual PacketSender* Create() const { + return new AdaptiveVideoSender(NULL, fps_, kbps_, ssrc_, frame_offset_); + } +}; + } // namespace bwe } // namespace testing } // namespace webrtc