From 3d7da88e062c29a001e2c3286bb126f853ed7d46 Mon Sep 17 00:00:00 2001 From: "stefan@webrtc.org" Date: Tue, 8 Jul 2014 13:59:46 +0000 Subject: [PATCH] Refactor ramp-up tests to have separate help files for the test classes, to make things more reusable. R=pbos@webrtc.org, tommi@webrtc.org Review URL: https://webrtc-codereview.appspot.com/18739004 git-svn-id: http://webrtc.googlecode.com/svn/trunk@6625 4adac7df-926f-26a2-2b94-8c16560cd09d --- webrtc/experiments.h | 10 - webrtc/video/rampup_tests.cc | 942 +++++++++++++++++------------------ webrtc/video/rampup_tests.h | 159 ++++++ webrtc/webrtc_tests.gypi | 1 + 4 files changed, 605 insertions(+), 507 deletions(-) create mode 100644 webrtc/video/rampup_tests.h diff --git a/webrtc/experiments.h b/webrtc/experiments.h index b03d248cd..3b019b484 100644 --- a/webrtc/experiments.h +++ b/webrtc/experiments.h @@ -14,16 +14,6 @@ #include "webrtc/typedefs.h" namespace webrtc { -struct PaddingStrategy { - PaddingStrategy() - : redundant_payloads(false) {} - explicit PaddingStrategy(bool redundant_payloads) - : redundant_payloads(redundant_payloads) {} - virtual ~PaddingStrategy() {} - - const bool redundant_payloads; -}; - struct RemoteBitrateEstimatorMinRate { RemoteBitrateEstimatorMinRate() : min_rate(30000) {} RemoteBitrateEstimatorMinRate(uint32_t min_rate) : min_rate(min_rate) {} diff --git a/webrtc/video/rampup_tests.cc b/webrtc/video/rampup_tests.cc index c83d37094..b6f63814a 100644 --- a/webrtc/video/rampup_tests.cc +++ b/webrtc/video/rampup_tests.cc @@ -7,548 +7,496 @@ * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ -#include - -#include -#include -#include #include "testing/gtest/include/gtest/gtest.h" - -#include "webrtc/call.h" -#include "webrtc/common.h" -#include "webrtc/experiments.h" -#include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" #include "webrtc/modules/rtp_rtcp/interface/receive_statistics.h" #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" #include "webrtc/modules/rtp_rtcp/interface/rtp_payload_registry.h" #include "webrtc/modules/rtp_rtcp/interface/rtp_rtcp.h" #include "webrtc/modules/rtp_rtcp/source/rtcp_utility.h" #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" -#include "webrtc/system_wrappers/interface/event_wrapper.h" -#include "webrtc/system_wrappers/interface/scoped_ptr.h" -#include "webrtc/test/call_test.h" -#include "webrtc/test/direct_transport.h" -#include "webrtc/test/encoder_settings.h" -#include "webrtc/test/fake_decoder.h" -#include "webrtc/test/fake_encoder.h" -#include "webrtc/test/frame_generator_capturer.h" #include "webrtc/test/testsupport/perf_test.h" -#include "webrtc/video/transport_adapter.h" +#include "webrtc/video/rampup_tests.h" namespace webrtc { - namespace { -static const int kTransmissionTimeOffsetExtensionId = 6; + static const int kMaxPacketSize = 1500; -static const unsigned int kSingleStreamTargetBps = 1000000; -class StreamObserver : public newapi::Transport, public RemoteBitrateObserver { - public: - typedef std::map BytesSentMap; - typedef std::map SsrcMap; - StreamObserver(const SsrcMap& rtx_media_ssrcs, - newapi::Transport* feedback_transport, - Clock* clock) - : clock_(clock), - test_done_(EventWrapper::Create()), - rtp_parser_(RtpHeaderParser::Create()), - feedback_transport_(feedback_transport), - receive_stats_(ReceiveStatistics::Create(clock)), - payload_registry_( - new RTPPayloadRegistry(RTPPayloadStrategy::CreateStrategy(false))), - crit_(CriticalSectionWrapper::CreateCriticalSection()), - expected_bitrate_bps_(0), - start_bitrate_bps_(0), - rtx_media_ssrcs_(rtx_media_ssrcs), - total_sent_(0), - padding_sent_(0), - rtx_media_sent_(0), - total_packets_sent_(0), - padding_packets_sent_(0), - rtx_media_packets_sent_(0) { - // Ideally we would only have to instantiate an RtcpSender, an - // RtpHeaderParser and a RemoteBitrateEstimator here, but due to the current - // state of the RTP module we need a full module and receive statistics to - // be able to produce an RTCP with REMB. - RtpRtcp::Configuration config; - config.receive_statistics = receive_stats_.get(); - feedback_transport_.Enable(); - config.outgoing_transport = &feedback_transport_; - rtp_rtcp_.reset(RtpRtcp::CreateRtpRtcp(config)); - rtp_rtcp_->SetREMBStatus(true); - rtp_rtcp_->SetRTCPStatus(kRtcpNonCompound); - rtp_parser_->RegisterRtpHeaderExtension(kRtpExtensionTransmissionTimeOffset, - kTransmissionTimeOffsetExtensionId); - AbsoluteSendTimeRemoteBitrateEstimatorFactory rbe_factory; - const uint32_t kRemoteBitrateEstimatorMinBitrateBps = 30000; - remote_bitrate_estimator_.reset( - rbe_factory.Create(this, clock, kMimdControl, - kRemoteBitrateEstimatorMinBitrateBps)); - } - - void set_expected_bitrate_bps(unsigned int expected_bitrate_bps) { - CriticalSectionScoped lock(crit_.get()); - expected_bitrate_bps_ = expected_bitrate_bps; - } - - void set_start_bitrate_bps(unsigned int start_bitrate_bps) { - CriticalSectionScoped lock(crit_.get()); - start_bitrate_bps_ = start_bitrate_bps; - } - - virtual void OnReceiveBitrateChanged(const std::vector& ssrcs, - unsigned int bitrate) OVERRIDE { - CriticalSectionScoped lock(crit_.get()); - assert(expected_bitrate_bps_ > 0); - if (start_bitrate_bps_ != 0) { - // For tests with an explicitly set start bitrate, verify the first - // bitrate estimate is close to the start bitrate and lower than the - // test target bitrate. This is to verify a call respects the configured - // start bitrate, but due to the BWE implementation we can't guarantee the - // first estimate really is as high as the start bitrate. - EXPECT_GT(bitrate, 0.9 * start_bitrate_bps_); - EXPECT_LT(bitrate, expected_bitrate_bps_); - start_bitrate_bps_ = 0; - } - if (bitrate >= expected_bitrate_bps_) { - // Just trigger if there was any rtx padding packet. - if (rtx_media_ssrcs_.empty() || rtx_media_sent_ > 0) { - TriggerTestDone(); - } - } - rtp_rtcp_->SetREMBData( - bitrate, static_cast(ssrcs.size()), &ssrcs[0]); - rtp_rtcp_->Process(); - } - - virtual bool SendRtp(const uint8_t* packet, size_t length) OVERRIDE { - CriticalSectionScoped lock(crit_.get()); - RTPHeader header; - EXPECT_TRUE(rtp_parser_->Parse(packet, length, &header)); - receive_stats_->IncomingPacket(header, length, false); - payload_registry_->SetIncomingPayloadType(header); - remote_bitrate_estimator_->IncomingPacket( - clock_->TimeInMilliseconds(), static_cast(length - 12), header); - if (remote_bitrate_estimator_->TimeUntilNextProcess() <= 0) { - remote_bitrate_estimator_->Process(); - } - total_sent_ += length; - padding_sent_ += header.paddingLength; - ++total_packets_sent_; - if (header.paddingLength > 0) - ++padding_packets_sent_; - if (rtx_media_ssrcs_.find(header.ssrc) != rtx_media_ssrcs_.end()) { - rtx_media_sent_ += length - header.headerLength - header.paddingLength; - if (header.paddingLength == 0) - ++rtx_media_packets_sent_; - uint8_t restored_packet[kMaxPacketSize]; - uint8_t* restored_packet_ptr = restored_packet; - int restored_length = static_cast(length); - payload_registry_->RestoreOriginalPacket(&restored_packet_ptr, - packet, - &restored_length, - rtx_media_ssrcs_[header.ssrc], - header); - length = restored_length; - EXPECT_TRUE(rtp_parser_->Parse(restored_packet, length, &header)); - } else { - rtp_rtcp_->SetRemoteSSRC(header.ssrc); - } - return true; - } - - virtual bool SendRtcp(const uint8_t* packet, size_t length) OVERRIDE { - return true; - } - - EventTypeWrapper Wait() { return test_done_->Wait(120 * 1000); } - - private: - void ReportResult(const std::string& measurement, - size_t value, - const std::string& units) { - webrtc::test::PrintResult( - measurement, "", - ::testing::UnitTest::GetInstance()->current_test_info()->name(), - value, units, false); - } - - void TriggerTestDone() EXCLUSIVE_LOCKS_REQUIRED(crit_) { - ReportResult("total-sent", total_sent_, "bytes"); - ReportResult("padding-sent", padding_sent_, "bytes"); - ReportResult("rtx-media-sent", rtx_media_sent_, "bytes"); - ReportResult("total-packets-sent", total_packets_sent_, "packets"); - ReportResult("padding-packets-sent", padding_packets_sent_, "packets"); - ReportResult("rtx-packets-sent", rtx_media_packets_sent_, "packets"); - test_done_->Set(); - } - - Clock* const clock_; - const scoped_ptr test_done_; - const scoped_ptr rtp_parser_; - scoped_ptr rtp_rtcp_; - internal::TransportAdapter feedback_transport_; - const scoped_ptr receive_stats_; - const scoped_ptr payload_registry_; - scoped_ptr remote_bitrate_estimator_; - - const scoped_ptr crit_; - unsigned int expected_bitrate_bps_ GUARDED_BY(crit_); - unsigned int start_bitrate_bps_ GUARDED_BY(crit_); - SsrcMap rtx_media_ssrcs_ GUARDED_BY(crit_); - size_t total_sent_ GUARDED_BY(crit_); - size_t padding_sent_ GUARDED_BY(crit_); - size_t rtx_media_sent_ GUARDED_BY(crit_); - int total_packets_sent_ GUARDED_BY(crit_); - int padding_packets_sent_ GUARDED_BY(crit_); - int rtx_media_packets_sent_ GUARDED_BY(crit_); -}; - -class LowRateStreamObserver : public test::DirectTransport, - public RemoteBitrateObserver, - public PacketReceiver { - public: - LowRateStreamObserver(newapi::Transport* feedback_transport, - Clock* clock, - size_t number_of_streams, - bool rtx_used) - : clock_(clock), - number_of_streams_(number_of_streams), - rtx_used_(rtx_used), - test_done_(EventWrapper::Create()), - rtp_parser_(RtpHeaderParser::Create()), - feedback_transport_(feedback_transport), - receive_stats_(ReceiveStatistics::Create(clock)), - crit_(CriticalSectionWrapper::CreateCriticalSection()), - send_stream_(NULL), - test_state_(kFirstRampup), - state_start_ms_(clock_->TimeInMilliseconds()), - interval_start_ms_(state_start_ms_), - last_remb_bps_(0), - sent_bytes_(0), - total_overuse_bytes_(0), - suspended_in_stats_(false) { - RtpRtcp::Configuration config; - config.receive_statistics = receive_stats_.get(); - feedback_transport_.Enable(); - config.outgoing_transport = &feedback_transport_; - rtp_rtcp_.reset(RtpRtcp::CreateRtpRtcp(config)); - rtp_rtcp_->SetREMBStatus(true); - rtp_rtcp_->SetRTCPStatus(kRtcpNonCompound); - rtp_parser_->RegisterRtpHeaderExtension(kRtpExtensionTransmissionTimeOffset, - kTransmissionTimeOffsetExtensionId); - AbsoluteSendTimeRemoteBitrateEstimatorFactory rbe_factory; - const uint32_t kRemoteBitrateEstimatorMinBitrateBps = 10000; - remote_bitrate_estimator_.reset( - rbe_factory.Create(this, clock, kMimdControl, - kRemoteBitrateEstimatorMinBitrateBps)); - forward_transport_config_.link_capacity_kbps = - kHighBandwidthLimitBps / 1000; - forward_transport_config_.queue_length = 100; // Something large. - test::DirectTransport::SetConfig(forward_transport_config_); - test::DirectTransport::SetReceiver(this); - } - - virtual void SetSendStream(const VideoSendStream* send_stream) { - CriticalSectionScoped lock(crit_.get()); - send_stream_ = send_stream; - } - - virtual void OnReceiveBitrateChanged(const std::vector& ssrcs, - unsigned int bitrate) { - CriticalSectionScoped lock(crit_.get()); - rtp_rtcp_->SetREMBData( - bitrate, static_cast(ssrcs.size()), &ssrcs[0]); - rtp_rtcp_->Process(); - last_remb_bps_ = bitrate; - } - - virtual bool SendRtp(const uint8_t* data, size_t length) OVERRIDE { - CriticalSectionScoped lock(crit_.get()); - sent_bytes_ += length; - int64_t now_ms = clock_->TimeInMilliseconds(); - if (now_ms > interval_start_ms_ + 1000) { // Let at least 1 second pass. - // Verify that the send rate was about right. - unsigned int average_rate_bps = static_cast(sent_bytes_) * - 8 * 1000 / (now_ms - interval_start_ms_); - // TODO(holmer): Why is this failing? - // EXPECT_LT(average_rate_bps, last_remb_bps_ * 1.1); - if (average_rate_bps > last_remb_bps_ * 1.1) { - total_overuse_bytes_ += - sent_bytes_ - - last_remb_bps_ / 8 * (now_ms - interval_start_ms_) / 1000; - } - EvolveTestState(average_rate_bps); - interval_start_ms_ = now_ms; - sent_bytes_ = 0; - } - return test::DirectTransport::SendRtp(data, length); - } - - virtual DeliveryStatus DeliverPacket(const uint8_t* packet, - size_t length) OVERRIDE { - CriticalSectionScoped lock(crit_.get()); - RTPHeader header; - EXPECT_TRUE(rtp_parser_->Parse(packet, length, &header)); - receive_stats_->IncomingPacket(header, length, false); - remote_bitrate_estimator_->IncomingPacket( - clock_->TimeInMilliseconds(), static_cast(length - 12), header); - if (remote_bitrate_estimator_->TimeUntilNextProcess() <= 0) { - remote_bitrate_estimator_->Process(); - } - suspended_in_stats_ = send_stream_->GetStats().suspended; - return DELIVERY_OK; - } - - virtual bool SendRtcp(const uint8_t* packet, size_t length) OVERRIDE { - return true; - } - - // Produces a string similar to "1stream_nortx", depending on the values of - // number_of_streams_ and rtx_used_; - std::string GetModifierString() { - std::string str("_"); - char temp_str[5]; - sprintf(temp_str, "%i", static_cast(number_of_streams_)); - str += std::string(temp_str); - str += "stream"; - str += (number_of_streams_ > 1 ? "s" : ""); - str += "_"; - str += (rtx_used_ ? "" : "no"); - str += "rtx"; - return str; - } - - // This method defines the state machine for the ramp up-down-up test. - void EvolveTestState(unsigned int bitrate_bps) { - int64_t now = clock_->TimeInMilliseconds(); - CriticalSectionScoped lock(crit_.get()); - assert(send_stream_ != NULL); - switch (test_state_) { - case kFirstRampup: { - EXPECT_FALSE(suspended_in_stats_); - if (bitrate_bps > kExpectedHighBitrateBps) { - // The first ramp-up has reached the target bitrate. Change the - // channel limit, and move to the next test state. - forward_transport_config_.link_capacity_kbps = - kLowBandwidthLimitBps / 1000; - test::DirectTransport::SetConfig(forward_transport_config_); - test_state_ = kLowRate; - webrtc::test::PrintResult("ramp_up_down_up", - GetModifierString(), - "first_rampup", - now - state_start_ms_, - "ms", - false); - state_start_ms_ = now; - interval_start_ms_ = now; - sent_bytes_ = 0; - } - break; - } - case kLowRate: { - if (bitrate_bps < kExpectedLowBitrateBps && suspended_in_stats_) { - // The ramp-down was successful. Change the channel limit back to a - // high value, and move to the next test state. - forward_transport_config_.link_capacity_kbps = - kHighBandwidthLimitBps / 1000; - test::DirectTransport::SetConfig(forward_transport_config_); - test_state_ = kSecondRampup; - webrtc::test::PrintResult("ramp_up_down_up", - GetModifierString(), - "rampdown", - now - state_start_ms_, - "ms", - false); - state_start_ms_ = now; - interval_start_ms_ = now; - sent_bytes_ = 0; - } - break; - } - case kSecondRampup: { - if (bitrate_bps > kExpectedHighBitrateBps && !suspended_in_stats_) { - webrtc::test::PrintResult("ramp_up_down_up", - GetModifierString(), - "second_rampup", - now - state_start_ms_, - "ms", - false); - webrtc::test::PrintResult("ramp_up_down_up", - GetModifierString(), - "total_overuse", - total_overuse_bytes_, - "bytes", - false); - test_done_->Set(); - } - break; - } - } - } - - EventTypeWrapper Wait() { - return test_done_->Wait(test::CallTest::kLongTimeoutMs); - } - - private: - static const unsigned int kHighBandwidthLimitBps = 80000; - static const unsigned int kExpectedHighBitrateBps = 60000; - static const unsigned int kLowBandwidthLimitBps = 20000; - static const unsigned int kExpectedLowBitrateBps = 20000; - enum TestStates { kFirstRampup, kLowRate, kSecondRampup }; - - Clock* const clock_; - const size_t number_of_streams_; - const bool rtx_used_; - const scoped_ptr test_done_; - const scoped_ptr rtp_parser_; - scoped_ptr rtp_rtcp_; - internal::TransportAdapter feedback_transport_; - const scoped_ptr receive_stats_; - scoped_ptr remote_bitrate_estimator_; - - scoped_ptr crit_; - const VideoSendStream* send_stream_ GUARDED_BY(crit_); - FakeNetworkPipe::Config forward_transport_config_ GUARDED_BY(crit_); - TestStates test_state_ GUARDED_BY(crit_); - int64_t state_start_ms_ GUARDED_BY(crit_); - int64_t interval_start_ms_ GUARDED_BY(crit_); - unsigned int last_remb_bps_ GUARDED_BY(crit_); - size_t sent_bytes_ GUARDED_BY(crit_); - size_t total_overuse_bytes_ GUARDED_BY(crit_); - bool suspended_in_stats_ GUARDED_BY(crit_); -}; +std::vector GenerateSsrcs(size_t num_streams, + uint32_t ssrc_offset) { + std::vector ssrcs; + for (size_t i = 0; i != num_streams; ++i) + ssrcs.push_back(static_cast(ssrc_offset + i)); + return ssrcs; +} } // namespace -class RampUpTest : public test::CallTest { - protected: - void RunRampUpTest(bool rtx, - size_t num_streams, - unsigned int start_bitrate_bps) { - std::vector ssrcs(GenerateSsrcs(num_streams, 100)); - std::vector rtx_ssrcs(GenerateSsrcs(num_streams, 200)); - StreamObserver::SsrcMap rtx_ssrc_map; - if (rtx) { - for (size_t i = 0; i < ssrcs.size(); ++i) - rtx_ssrc_map[rtx_ssrcs[i]] = ssrcs[i]; +StreamObserver::StreamObserver(const SsrcMap& rtx_media_ssrcs, + newapi::Transport* feedback_transport, + Clock* clock, + RemoteBitrateEstimatorFactory* rbe_factory, + RateControlType control_type) + : clock_(clock), + test_done_(EventWrapper::Create()), + rtp_parser_(RtpHeaderParser::Create()), + feedback_transport_(feedback_transport), + receive_stats_(ReceiveStatistics::Create(clock)), + payload_registry_( + new RTPPayloadRegistry(RTPPayloadStrategy::CreateStrategy(false))), + crit_(CriticalSectionWrapper::CreateCriticalSection()), + expected_bitrate_bps_(0), + start_bitrate_bps_(0), + rtx_media_ssrcs_(rtx_media_ssrcs), + total_sent_(0), + padding_sent_(0), + rtx_media_sent_(0), + total_packets_sent_(0), + padding_packets_sent_(0), + rtx_media_packets_sent_(0), + test_start_ms_(clock_->TimeInMilliseconds()), + ramp_up_finished_ms_(0) { + // Ideally we would only have to instantiate an RtcpSender, an + // RtpHeaderParser and a RemoteBitrateEstimator here, but due to the current + // state of the RTP module we need a full module and receive statistics to + // be able to produce an RTCP with REMB. + RtpRtcp::Configuration config; + config.receive_statistics = receive_stats_.get(); + feedback_transport_.Enable(); + config.outgoing_transport = &feedback_transport_; + rtp_rtcp_.reset(RtpRtcp::CreateRtpRtcp(config)); + rtp_rtcp_->SetREMBStatus(true); + rtp_rtcp_->SetRTCPStatus(kRtcpNonCompound); + rtp_parser_->RegisterRtpHeaderExtension(kRtpExtensionAbsoluteSendTime, + kAbsSendTimeExtensionId); + rtp_parser_->RegisterRtpHeaderExtension(kRtpExtensionTransmissionTimeOffset, + kTransmissionTimeOffsetExtensionId); + const uint32_t kRemoteBitrateEstimatorMinBitrateBps = 30000; + remote_bitrate_estimator_.reset( + rbe_factory->Create(this, clock, control_type, + kRemoteBitrateEstimatorMinBitrateBps)); +} + +void StreamObserver::set_expected_bitrate_bps( + unsigned int expected_bitrate_bps) { + CriticalSectionScoped lock(crit_.get()); + expected_bitrate_bps_ = expected_bitrate_bps; +} + +void StreamObserver::set_start_bitrate_bps(unsigned int start_bitrate_bps) { + CriticalSectionScoped lock(crit_.get()); + start_bitrate_bps_ = start_bitrate_bps; +} + +void StreamObserver::OnReceiveBitrateChanged( + const std::vector& ssrcs, unsigned int bitrate) { + CriticalSectionScoped lock(crit_.get()); + assert(expected_bitrate_bps_ > 0); + if (start_bitrate_bps_ != 0) { + // For tests with an explicitly set start bitrate, verify the first + // bitrate estimate is close to the start bitrate and lower than the + // test target bitrate. This is to verify a call respects the configured + // start bitrate, but due to the BWE implementation we can't guarantee the + // first estimate really is as high as the start bitrate. + EXPECT_GT(bitrate, 0.9 * start_bitrate_bps_); + EXPECT_LT(bitrate, expected_bitrate_bps_); + start_bitrate_bps_ = 0; + } + if (bitrate >= expected_bitrate_bps_) { + ramp_up_finished_ms_ = clock_->TimeInMilliseconds(); + // Just trigger if there was any rtx padding packet. + if (rtx_media_ssrcs_.empty() || rtx_media_sent_ > 0) { + TriggerTestDone(); } - test::DirectTransport receiver_transport; - StreamObserver stream_observer(rtx_ssrc_map, - &receiver_transport, - Clock::GetRealTimeClock()); + } + rtp_rtcp_->SetREMBData( + bitrate, static_cast(ssrcs.size()), &ssrcs[0]); + rtp_rtcp_->Process(); +} - Call::Config call_config(&stream_observer); - if (start_bitrate_bps != 0) { - call_config.start_bitrate_bps = start_bitrate_bps; - stream_observer.set_start_bitrate_bps(start_bitrate_bps); +bool StreamObserver::SendRtp(const uint8_t* packet, size_t length) { + CriticalSectionScoped lock(crit_.get()); + RTPHeader header; + EXPECT_TRUE(rtp_parser_->Parse(packet, static_cast(length), &header)); + receive_stats_->IncomingPacket(header, length, false); + payload_registry_->SetIncomingPayloadType(header); + remote_bitrate_estimator_->IncomingPacket( + clock_->TimeInMilliseconds(), static_cast(length - 12), header); + if (remote_bitrate_estimator_->TimeUntilNextProcess() <= 0) { + remote_bitrate_estimator_->Process(); + } + total_sent_ += length; + padding_sent_ += header.paddingLength; + ++total_packets_sent_; + if (header.paddingLength > 0) + ++padding_packets_sent_; + if (rtx_media_ssrcs_.find(header.ssrc) != rtx_media_ssrcs_.end()) { + rtx_media_sent_ += length - header.headerLength - header.paddingLength; + if (header.paddingLength == 0) + ++rtx_media_packets_sent_; + uint8_t restored_packet[kMaxPacketSize]; + uint8_t* restored_packet_ptr = restored_packet; + int restored_length = static_cast(length); + payload_registry_->RestoreOriginalPacket(&restored_packet_ptr, + packet, + &restored_length, + rtx_media_ssrcs_[header.ssrc], + header); + length = restored_length; + EXPECT_TRUE(rtp_parser_->Parse( + restored_packet, static_cast(length), &header)); + } else { + rtp_rtcp_->SetRemoteSSRC(header.ssrc); + } + return true; +} + +bool StreamObserver::SendRtcp(const uint8_t* packet, size_t length) { + return true; +} + +EventTypeWrapper StreamObserver::Wait() { return test_done_->Wait(120 * 1000); } + +void StreamObserver::ReportResult(const std::string& measurement, + size_t value, + const std::string& units) { + webrtc::test::PrintResult( + measurement, "", + ::testing::UnitTest::GetInstance()->current_test_info()->name(), + value, units, false); +} + +void StreamObserver::TriggerTestDone() EXCLUSIVE_LOCKS_REQUIRED(crit_) { + ReportResult("ramp-up-total-sent", total_sent_, "bytes"); + ReportResult("ramp-up-padding-sent", padding_sent_, "bytes"); + ReportResult("ramp-up-rtx-media-sent", rtx_media_sent_, "bytes"); + ReportResult("ramp-up-total-packets-sent", total_packets_sent_, "packets"); + ReportResult("ramp-up-padding-packets-sent", + padding_packets_sent_, + "packets"); + ReportResult("ramp-up-rtx-packets-sent", + rtx_media_packets_sent_, + "packets"); + ReportResult("ramp-up-time", + ramp_up_finished_ms_ - test_start_ms_, + "milliseconds"); + test_done_->Set(); +} + +LowRateStreamObserver::LowRateStreamObserver( + newapi::Transport* feedback_transport, + Clock* clock, + size_t number_of_streams, + bool rtx_used) + : clock_(clock), + number_of_streams_(number_of_streams), + rtx_used_(rtx_used), + test_done_(EventWrapper::Create()), + rtp_parser_(RtpHeaderParser::Create()), + feedback_transport_(feedback_transport), + receive_stats_(ReceiveStatistics::Create(clock)), + crit_(CriticalSectionWrapper::CreateCriticalSection()), + send_stream_(NULL), + test_state_(kFirstRampup), + state_start_ms_(clock_->TimeInMilliseconds()), + interval_start_ms_(state_start_ms_), + last_remb_bps_(0), + sent_bytes_(0), + total_overuse_bytes_(0), + suspended_in_stats_(false) { + RtpRtcp::Configuration config; + config.receive_statistics = receive_stats_.get(); + feedback_transport_.Enable(); + config.outgoing_transport = &feedback_transport_; + rtp_rtcp_.reset(RtpRtcp::CreateRtpRtcp(config)); + rtp_rtcp_->SetREMBStatus(true); + rtp_rtcp_->SetRTCPStatus(kRtcpNonCompound); + rtp_parser_->RegisterRtpHeaderExtension(kRtpExtensionTransmissionTimeOffset, + kTransmissionTimeOffsetExtensionId); + AbsoluteSendTimeRemoteBitrateEstimatorFactory rbe_factory; + const uint32_t kRemoteBitrateEstimatorMinBitrateBps = 10000; + remote_bitrate_estimator_.reset( + rbe_factory.Create(this, clock, kMimdControl, + kRemoteBitrateEstimatorMinBitrateBps)); + forward_transport_config_.link_capacity_kbps = + kHighBandwidthLimitBps / 1000; + forward_transport_config_.queue_length = 100; // Something large. + test::DirectTransport::SetConfig(forward_transport_config_); + test::DirectTransport::SetReceiver(this); +} + +void LowRateStreamObserver::SetSendStream(const VideoSendStream* send_stream) { + CriticalSectionScoped lock(crit_.get()); + send_stream_ = send_stream; +} + +void LowRateStreamObserver::OnReceiveBitrateChanged( + const std::vector& ssrcs, + unsigned int bitrate) { + CriticalSectionScoped lock(crit_.get()); + rtp_rtcp_->SetREMBData( + bitrate, static_cast(ssrcs.size()), &ssrcs[0]); + rtp_rtcp_->Process(); + last_remb_bps_ = bitrate; +} + +bool LowRateStreamObserver::SendRtp(const uint8_t* data, size_t length) { + CriticalSectionScoped lock(crit_.get()); + sent_bytes_ += length; + int64_t now_ms = clock_->TimeInMilliseconds(); + if (now_ms > interval_start_ms_ + 1000) { // Let at least 1 second pass. + // Verify that the send rate was about right. + unsigned int average_rate_bps = static_cast(sent_bytes_) * + 8 * 1000 / (now_ms - interval_start_ms_); + // TODO(holmer): Why is this failing? + // EXPECT_LT(average_rate_bps, last_remb_bps_ * 1.1); + if (average_rate_bps > last_remb_bps_ * 1.1) { + total_overuse_bytes_ += + sent_bytes_ - + last_remb_bps_ / 8 * (now_ms - interval_start_ms_) / 1000; } + EvolveTestState(average_rate_bps); + interval_start_ms_ = now_ms; + sent_bytes_ = 0; + } + return test::DirectTransport::SendRtp(data, length); +} - CreateSenderCall(call_config); - CreateSendConfig(num_streams); +PacketReceiver::DeliveryStatus LowRateStreamObserver::DeliverPacket( + const uint8_t* packet, size_t length) { + CriticalSectionScoped lock(crit_.get()); + RTPHeader header; + EXPECT_TRUE(rtp_parser_->Parse(packet, static_cast(length), &header)); + receive_stats_->IncomingPacket(header, length, false); + remote_bitrate_estimator_->IncomingPacket( + clock_->TimeInMilliseconds(), static_cast(length - 12), header); + if (remote_bitrate_estimator_->TimeUntilNextProcess() <= 0) { + remote_bitrate_estimator_->Process(); + } + suspended_in_stats_ = send_stream_->GetStats().suspended; + return DELIVERY_OK; +} - receiver_transport.SetReceiver(sender_call_->Receiver()); +bool LowRateStreamObserver::SendRtcp(const uint8_t* packet, size_t length) { + return true; +} - if (num_streams == 1) { - video_streams_[0].target_bitrate_bps = 2000000; - video_streams_[0].max_bitrate_bps = 2000000; - } +std::string LowRateStreamObserver::GetModifierString() { + std::string str("_"); + char temp_str[5]; + sprintf(temp_str, "%i", + static_cast(number_of_streams_)); + str += std::string(temp_str); + str += "stream"; + str += (number_of_streams_ > 1 ? "s" : ""); + str += "_"; + str += (rtx_used_ ? "" : "no"); + str += "rtx"; + return str; +} - send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs; - send_config_.rtp.ssrcs = ssrcs; - if (rtx) { - send_config_.rtp.rtx.payload_type = kSendRtxPayloadType; - send_config_.rtp.rtx.ssrcs = rtx_ssrcs; - send_config_.rtp.rtx.pad_with_redundant_payloads = true; - } - send_config_.rtp.extensions.push_back(RtpExtension( - RtpExtension::kTOffset, kTransmissionTimeOffsetExtensionId)); - - if (num_streams == 1) { - // For single stream rampup until 1mbps - stream_observer.set_expected_bitrate_bps(kSingleStreamTargetBps); - } else { - // For multi stream rampup until all streams are being sent. That means - // enough birate to send all the target streams plus the min bitrate of - // the last one. - int expected_bitrate_bps = video_streams_.back().min_bitrate_bps; - for (size_t i = 0; i < video_streams_.size() - 1; ++i) { - expected_bitrate_bps += video_streams_[i].target_bitrate_bps; +void LowRateStreamObserver::EvolveTestState(unsigned int bitrate_bps) { + int64_t now = clock_->TimeInMilliseconds(); + CriticalSectionScoped lock(crit_.get()); + assert(send_stream_ != NULL); + switch (test_state_) { + case kFirstRampup: { + EXPECT_FALSE(suspended_in_stats_); + if (bitrate_bps > kExpectedHighBitrateBps) { + // The first ramp-up has reached the target bitrate. Change the + // channel limit, and move to the next test state. + forward_transport_config_.link_capacity_kbps = + kLowBandwidthLimitBps / 1000; + test::DirectTransport::SetConfig(forward_transport_config_); + test_state_ = kLowRate; + webrtc::test::PrintResult("ramp_up_down_up", + GetModifierString(), + "first_rampup", + now - state_start_ms_, + "ms", + false); + state_start_ms_ = now; + interval_start_ms_ = now; + sent_bytes_ = 0; } - stream_observer.set_expected_bitrate_bps(expected_bitrate_bps); + break; } + case kLowRate: { + if (bitrate_bps < kExpectedLowBitrateBps && suspended_in_stats_) { + // The ramp-down was successful. Change the channel limit back to a + // high value, and move to the next test state. + forward_transport_config_.link_capacity_kbps = + kHighBandwidthLimitBps / 1000; + test::DirectTransport::SetConfig(forward_transport_config_); + test_state_ = kSecondRampup; + webrtc::test::PrintResult("ramp_up_down_up", + GetModifierString(), + "rampdown", + now - state_start_ms_, + "ms", + false); + state_start_ms_ = now; + interval_start_ms_ = now; + sent_bytes_ = 0; + } + break; + } + case kSecondRampup: { + if (bitrate_bps > kExpectedHighBitrateBps && !suspended_in_stats_) { + webrtc::test::PrintResult("ramp_up_down_up", + GetModifierString(), + "second_rampup", + now - state_start_ms_, + "ms", + false); + webrtc::test::PrintResult("ramp_up_down_up", + GetModifierString(), + "total_overuse", + total_overuse_bytes_, + "bytes", + false); + test_done_->Set(); + } + break; + } + } +} - CreateStreams(); - CreateFrameGeneratorCapturer(); +EventTypeWrapper LowRateStreamObserver::Wait() { + return test_done_->Wait(test::CallTest::kLongTimeoutMs); +} - Start(); - - EXPECT_EQ(kEventSignaled, stream_observer.Wait()); - - Stop(); - DestroyStreams(); +void RampUpTest::RunRampUpTest(bool rtx, + size_t num_streams, + unsigned int start_bitrate_bps, + const std::string& extension_type) { + std::vector ssrcs(GenerateSsrcs(num_streams, 100)); + std::vector rtx_ssrcs(GenerateSsrcs(num_streams, 200)); + StreamObserver::SsrcMap rtx_ssrc_map; + if (rtx) { + for (size_t i = 0; i < ssrcs.size(); ++i) + rtx_ssrc_map[rtx_ssrcs[i]] = ssrcs[i]; } - void RunRampUpDownUpTest(size_t number_of_streams, bool rtx) { - std::vector ssrcs; - for (size_t i = 0; i < number_of_streams; ++i) - ssrcs.push_back(static_cast(i + 1)); - test::DirectTransport receiver_transport; - LowRateStreamObserver stream_observer( - &receiver_transport, Clock::GetRealTimeClock(), number_of_streams, rtx); + CreateSendConfig(num_streams); - Call::Config call_config(&stream_observer); - webrtc::Config webrtc_config; - call_config.webrtc_config = &webrtc_config; - webrtc_config.Set(new PaddingStrategy(rtx)); - CreateSenderCall(call_config); - receiver_transport.SetReceiver(sender_call_->Receiver()); - - CreateSendConfig(number_of_streams); - - send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs; + scoped_ptr rbe_factory; + RateControlType control_type; + if (extension_type == RtpExtension::kAbsSendTime) { + control_type = kAimdControl; + rbe_factory.reset(new AbsoluteSendTimeRemoteBitrateEstimatorFactory); send_config_.rtp.extensions.push_back(RtpExtension( - RtpExtension::kTOffset, kTransmissionTimeOffsetExtensionId)); - send_config_.suspend_below_min_bitrate = true; - - CreateStreams(); - stream_observer.SetSendStream(send_stream_); - - CreateFrameGeneratorCapturer(); - - Start(); - - EXPECT_EQ(kEventSignaled, stream_observer.Wait()); - - Stop(); - DestroyStreams(); + extension_type.c_str(), kAbsSendTimeExtensionId)); + } else { + control_type = kMimdControl; + rbe_factory.reset(new RemoteBitrateEstimatorFactory); + send_config_.rtp.extensions.push_back(RtpExtension( + extension_type.c_str(), kTransmissionTimeOffsetExtensionId)); } - private: - std::vector GenerateSsrcs(size_t num_streams, - uint32_t ssrc_offset) { - std::vector ssrcs; - for (size_t i = 0; i != num_streams; ++i) - ssrcs.push_back(static_cast(ssrc_offset + i)); - return ssrcs; + test::DirectTransport receiver_transport; + StreamObserver stream_observer(rtx_ssrc_map, + &receiver_transport, + Clock::GetRealTimeClock(), + rbe_factory.get(), + control_type); + + Call::Config call_config(&stream_observer); + if (start_bitrate_bps != 0) { + call_config.start_bitrate_bps = start_bitrate_bps; + stream_observer.set_start_bitrate_bps(start_bitrate_bps); } -}; + + CreateSenderCall(call_config); + + receiver_transport.SetReceiver(sender_call_->Receiver()); + + if (num_streams == 1) { + video_streams_[0].target_bitrate_bps = 2000000; + video_streams_[0].max_bitrate_bps = 2000000; + } + + send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs; + send_config_.rtp.ssrcs = ssrcs; + if (rtx) { + send_config_.rtp.rtx.payload_type = kSendRtxPayloadType; + send_config_.rtp.rtx.ssrcs = rtx_ssrcs; + send_config_.rtp.rtx.pad_with_redundant_payloads = true; + } + + if (num_streams == 1) { + // For single stream rampup until 1mbps + stream_observer.set_expected_bitrate_bps(kSingleStreamTargetBps); + } else { + // For multi stream rampup until all streams are being sent. That means + // enough birate to send all the target streams plus the min bitrate of + // the last one. + int expected_bitrate_bps = video_streams_.back().min_bitrate_bps; + for (size_t i = 0; i < video_streams_.size() - 1; ++i) { + expected_bitrate_bps += video_streams_[i].target_bitrate_bps; + } + stream_observer.set_expected_bitrate_bps(expected_bitrate_bps); + } + + CreateStreams(); + CreateFrameGeneratorCapturer(); + + Start(); + + EXPECT_EQ(kEventSignaled, stream_observer.Wait()); + + Stop(); + DestroyStreams(); +} + +void RampUpTest::RunRampUpDownUpTest(size_t number_of_streams, bool rtx) { + test::DirectTransport receiver_transport; + LowRateStreamObserver stream_observer( + &receiver_transport, Clock::GetRealTimeClock(), number_of_streams, rtx); + + Call::Config call_config(&stream_observer); + CreateSenderCall(call_config); + receiver_transport.SetReceiver(sender_call_->Receiver()); + + CreateSendConfig(number_of_streams); + + send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs; + send_config_.rtp.extensions.push_back(RtpExtension( + RtpExtension::kTOffset, kTransmissionTimeOffsetExtensionId)); + send_config_.suspend_below_min_bitrate = true; + if (rtx) { + send_config_.rtp.rtx.payload_type = kSendRtxPayloadType; + send_config_.rtp.rtx.ssrcs = GenerateSsrcs(number_of_streams, 200); + send_config_.rtp.rtx.pad_with_redundant_payloads = true; + } + + CreateStreams(); + stream_observer.SetSendStream(send_stream_); + + CreateFrameGeneratorCapturer(); + + Start(); + + EXPECT_EQ(kEventSignaled, stream_observer.Wait()); + + Stop(); + DestroyStreams(); +} TEST_F(RampUpTest, SingleStream) { - RunRampUpTest(false, 1, 0); + RunRampUpTest(false, 1, 0, RtpExtension::kTOffset); } TEST_F(RampUpTest, Simulcast) { - RunRampUpTest(false, 3, 0); + RunRampUpTest(false, 3, 0, RtpExtension::kTOffset); } TEST_F(RampUpTest, SimulcastWithRtx) { - RunRampUpTest(true, 3, 0); + RunRampUpTest(true, 3, 0, RtpExtension::kTOffset); } TEST_F(RampUpTest, SingleStreamWithHighStartBitrate) { - RunRampUpTest(false, 1, 0.9 * kSingleStreamTargetBps); + RunRampUpTest(false, 1, 0.9 * kSingleStreamTargetBps, RtpExtension::kTOffset); } TEST_F(RampUpTest, UpDownUpOneStream) { RunRampUpDownUpTest(1, false); } diff --git a/webrtc/video/rampup_tests.h b/webrtc/video/rampup_tests.h new file mode 100644 index 000000000..69399b419 --- /dev/null +++ b/webrtc/video/rampup_tests.h @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_VIDEO_RAMPUP_TESTS_H_ +#define WEBRTC_VIDEO_RAMPUP_TESTS_H_ + +#include +#include +#include + +#include "webrtc/call.h" +#include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" +#include "webrtc/system_wrappers/interface/event_wrapper.h" +#include "webrtc/system_wrappers/interface/scoped_ptr.h" +#include "webrtc/test/call_test.h" +#include "webrtc/video/transport_adapter.h" + +namespace webrtc { + +static const int kTransmissionTimeOffsetExtensionId = 6; +static const int kAbsSendTimeExtensionId = 7; +static const unsigned int kSingleStreamTargetBps = 1000000; + +class Clock; +class CriticalSectionWrapper; +class ReceiveStatistics; +class RtpHeaderParser; +class RTPPayloadRegistry; +class RtpRtcp; + +class StreamObserver : public newapi::Transport, public RemoteBitrateObserver { + public: + typedef std::map BytesSentMap; + typedef std::map SsrcMap; + StreamObserver(const SsrcMap& rtx_media_ssrcs, + newapi::Transport* feedback_transport, + Clock* clock, + RemoteBitrateEstimatorFactory* rbe_factory, + RateControlType control_type); + + void set_expected_bitrate_bps(unsigned int expected_bitrate_bps); + + void set_start_bitrate_bps(unsigned int start_bitrate_bps); + + virtual void OnReceiveBitrateChanged(const std::vector& ssrcs, + unsigned int bitrate) OVERRIDE; + + virtual bool SendRtp(const uint8_t* packet, size_t length) OVERRIDE; + + virtual bool SendRtcp(const uint8_t* packet, size_t length) OVERRIDE; + + EventTypeWrapper Wait(); + + private: + void ReportResult(const std::string& measurement, + size_t value, + const std::string& units); + void TriggerTestDone() EXCLUSIVE_LOCKS_REQUIRED(crit_); + + Clock* const clock_; + const scoped_ptr test_done_; + const scoped_ptr rtp_parser_; + scoped_ptr rtp_rtcp_; + internal::TransportAdapter feedback_transport_; + const scoped_ptr receive_stats_; + const scoped_ptr payload_registry_; + scoped_ptr remote_bitrate_estimator_; + + const scoped_ptr crit_; + unsigned int expected_bitrate_bps_ GUARDED_BY(crit_); + unsigned int start_bitrate_bps_ GUARDED_BY(crit_); + SsrcMap rtx_media_ssrcs_ GUARDED_BY(crit_); + size_t total_sent_ GUARDED_BY(crit_); + size_t padding_sent_ GUARDED_BY(crit_); + size_t rtx_media_sent_ GUARDED_BY(crit_); + int total_packets_sent_ GUARDED_BY(crit_); + int padding_packets_sent_ GUARDED_BY(crit_); + int rtx_media_packets_sent_ GUARDED_BY(crit_); + int64_t test_start_ms_ GUARDED_BY(crit_); + int64_t ramp_up_finished_ms_ GUARDED_BY(crit_); +}; + +class LowRateStreamObserver : public test::DirectTransport, + public RemoteBitrateObserver, + public PacketReceiver { + public: + LowRateStreamObserver(newapi::Transport* feedback_transport, + Clock* clock, + size_t number_of_streams, + bool rtx_used); + + virtual void SetSendStream(const VideoSendStream* send_stream); + + virtual void OnReceiveBitrateChanged(const std::vector& ssrcs, + unsigned int bitrate); + + virtual bool SendRtp(const uint8_t* data, size_t length) OVERRIDE; + + virtual DeliveryStatus DeliverPacket(const uint8_t* packet, + size_t length) OVERRIDE; + + virtual bool SendRtcp(const uint8_t* packet, size_t length) OVERRIDE; + + // Produces a string similar to "1stream_nortx", depending on the values of + // number_of_streams_ and rtx_used_; + std::string GetModifierString(); + + // This method defines the state machine for the ramp up-down-up test. + void EvolveTestState(unsigned int bitrate_bps); + + EventTypeWrapper Wait(); + + private: + static const unsigned int kHighBandwidthLimitBps = 80000; + static const unsigned int kExpectedHighBitrateBps = 60000; + static const unsigned int kLowBandwidthLimitBps = 20000; + static const unsigned int kExpectedLowBitrateBps = 20000; + enum TestStates { kFirstRampup, kLowRate, kSecondRampup }; + + Clock* const clock_; + const size_t number_of_streams_; + const bool rtx_used_; + const scoped_ptr test_done_; + const scoped_ptr rtp_parser_; + scoped_ptr rtp_rtcp_; + internal::TransportAdapter feedback_transport_; + const scoped_ptr receive_stats_; + scoped_ptr remote_bitrate_estimator_; + + scoped_ptr crit_; + const VideoSendStream* send_stream_ GUARDED_BY(crit_); + FakeNetworkPipe::Config forward_transport_config_ GUARDED_BY(crit_); + TestStates test_state_ GUARDED_BY(crit_); + int64_t state_start_ms_ GUARDED_BY(crit_); + int64_t interval_start_ms_ GUARDED_BY(crit_); + unsigned int last_remb_bps_ GUARDED_BY(crit_); + size_t sent_bytes_ GUARDED_BY(crit_); + size_t total_overuse_bytes_ GUARDED_BY(crit_); + bool suspended_in_stats_ GUARDED_BY(crit_); +}; + +class RampUpTest : public test::CallTest { + protected: + void RunRampUpTest(bool rtx, + size_t num_streams, + unsigned int start_bitrate_bps, + const std::string& extension_type); + + void RunRampUpDownUpTest(size_t number_of_streams, bool rtx); +}; +} // namespace webrtc +#endif // WEBRTC_VIDEO_RAMPUP_TESTS_H_ diff --git a/webrtc/webrtc_tests.gypi b/webrtc/webrtc_tests.gypi index 85a7e733b..78b199f09 100644 --- a/webrtc/webrtc_tests.gypi +++ b/webrtc/webrtc_tests.gypi @@ -79,6 +79,7 @@ 'video/call_perf_tests.cc', 'video/full_stack.cc', 'video/rampup_tests.cc', + 'video/rampup_tests.h', ], 'dependencies': [ '<(DEPTH)/testing/gtest.gyp:gtest',