From cb89c6f91491e683e47e5505536c154c905d5194 Mon Sep 17 00:00:00 2001 From: "bjornv@webrtc.org" Date: Tue, 5 Jun 2012 12:25:35 +0000 Subject: [PATCH] Revert 2363 - Refactoring the receive-side bandwidth estimation into its own module. Each REMB group has one remote_bitrate_estimator object. For now the estimator keeps one estimate for every SSRC. In a later commit this will be unified and one estimate will be used for all SSRC in one group. BUG= TEST= Review URL: https://webrtc-codereview.appspot.com/605007 TBR=stefan@webrtc.org Review URL: https://webrtc-codereview.appspot.com/634006 git-svn-id: http://webrtc.googlecode.com/svn/trunk@2366 4adac7df-926f-26a2-2b94-8c16560cd09d --- src/modules/modules.gyp | 2 +- src/modules/rtp_rtcp/interface/rtp_rtcp.h | 10 +- .../rtp_rtcp/interface/rtp_rtcp_defines.h | 11 + src/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h | 9 +- src/modules/rtp_rtcp/source/Bitrate.h | 28 +- src/modules/rtp_rtcp/source/bitrate.cc | 75 +++ src/modules/rtp_rtcp/source/bwe_defines.h | 56 ++ .../source/mock/mock_rtp_receiver_video.h | 6 - .../rtp_rtcp/source/overuse_detector.cc | 423 +++++++++++++++ .../rtp_rtcp/source/overuse_detector.h | 87 ++++ .../rtp_rtcp/source/remote_rate_control.cc | 483 ++++++++++++++++++ .../rtp_rtcp/source/remote_rate_control.h | 82 +++ .../source/rtcp_format_remb_unittest.cc | 10 +- .../rtp_rtcp/source/rtcp_receiver_unittest.cc | 11 +- src/modules/rtp_rtcp/source/rtcp_sender.cc | 62 ++- src/modules/rtp_rtcp/source/rtcp_sender.h | 24 +- .../rtp_rtcp/source/rtcp_sender_unittest.cc | 18 +- src/modules/rtp_rtcp/source/rtp_receiver.cc | 4 +- src/modules/rtp_rtcp/source/rtp_receiver.h | 1 - .../rtp_rtcp/source/rtp_receiver_video.cc | 51 +- .../rtp_rtcp/source/rtp_receiver_video.h | 17 +- src/modules/rtp_rtcp/source/rtp_rtcp.gypi | 5 +- src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc | 49 +- src/modules/rtp_rtcp/source/rtp_rtcp_impl.h | 4 +- src/video_engine/vie_channel.cc | 4 +- src/video_engine/vie_channel.h | 3 +- src/video_engine/vie_channel_group.cc | 17 +- src/video_engine/vie_channel_group.h | 9 +- src/video_engine/vie_channel_manager.cc | 25 +- src/video_engine/vie_channel_manager.h | 2 +- src/video_engine/vie_remb.h | 3 +- 31 files changed, 1455 insertions(+), 136 deletions(-) create mode 100644 src/modules/rtp_rtcp/source/bwe_defines.h create mode 100644 src/modules/rtp_rtcp/source/overuse_detector.cc create mode 100644 src/modules/rtp_rtcp/source/overuse_detector.h create mode 100644 src/modules/rtp_rtcp/source/remote_rate_control.cc create mode 100644 src/modules/rtp_rtcp/source/remote_rate_control.h diff --git a/src/modules/modules.gyp b/src/modules/modules.gyp index d62bee4b1..5e6471509 100644 --- a/src/modules/modules.gyp +++ b/src/modules/modules.gyp @@ -28,7 +28,6 @@ 'audio_processing/utility/util.gypi', 'bitrate_controller/bitrate_controller.gypi', 'media_file/source/media_file.gypi', - 'remote_bitrate_estimator/remote_bitrate_estimator.gypi', 'udp_transport/source/udp_transport.gypi', 'utility/source/utility.gypi', 'video_coding/codecs/i420/main/source/i420.gypi', @@ -48,6 +47,7 @@ 'audio_coding/codecs/iSAC/isacfix_test.gypi', 'audio_processing/apm_tests.gypi', 'rtp_rtcp/source/rtp_rtcp_tests.gypi', + 'rtp_rtcp/test/test_bwe/test_bwe.gypi', 'rtp_rtcp/test/testFec/test_fec.gypi', 'rtp_rtcp/test/testAPI/test_api.gypi', 'video_coding/main/source/video_coding_test.gypi', diff --git a/src/modules/rtp_rtcp/interface/rtp_rtcp.h b/src/modules/rtp_rtcp/interface/rtp_rtcp.h index 7db981088..c1f9a4a2a 100644 --- a/src/modules/rtp_rtcp/interface/rtp_rtcp.h +++ b/src/modules/rtp_rtcp/interface/rtp_rtcp.h @@ -18,8 +18,6 @@ namespace webrtc { // forward declaration -class RemoteBitrateEstimator; -class RemoteBitrateObserver; class Transport; class RtpRtcp : public Module { @@ -37,7 +35,7 @@ class RtpRtcp : public Module { intra_frame_callback(NULL), bandwidth_callback(NULL), audio_messages(NULL), - remote_bitrate_estimator(NULL) { + bitrate_observer(NULL) { } /* id - Unique identifier of this RTP/RTCP module object * audio - True for a audio version of the RTP/RTCP module @@ -56,8 +54,8 @@ class RtpRtcp : public Module { * bandwidth_callback - Called when we receive a changed estimate from * the receiver of out stream. * audio_messages - Telehone events. - * remote_bitrate_estimator - Estimates the bandwidth available for a set of - * streams from the same client. + * bitrate_observer - Called when the estimate of the incoming RTP + * stream changes. */ int32_t id; bool audio; @@ -70,7 +68,7 @@ class RtpRtcp : public Module { RtcpIntraFrameObserver* intra_frame_callback; RtcpBandwidthObserver* bandwidth_callback; RtpAudioFeedback* audio_messages; - RemoteBitrateEstimator* remote_bitrate_estimator; + RtpRemoteBitrateObserver* bitrate_observer; }; /* * Create a RTP/RTCP module object using the system clock. diff --git a/src/modules/rtp_rtcp/interface/rtp_rtcp_defines.h b/src/modules/rtp_rtcp/interface/rtp_rtcp_defines.h index 6bc787ee7..110b08c2b 100644 --- a/src/modules/rtp_rtcp/interface/rtp_rtcp_defines.h +++ b/src/modules/rtp_rtcp/interface/rtp_rtcp_defines.h @@ -251,5 +251,16 @@ class RtpRtcpClock { virtual void CurrentNTP(WebRtc_UWord32& secs, WebRtc_UWord32& frac) = 0; }; +// RtpReceiveBitrateUpdate is used to signal changes in bitrate estimates for +// the incoming stream. +class RtpRemoteBitrateObserver { + public: + // Called when a receive channel has a new bitrate estimate for the incoming + // stream. + virtual void OnReceiveBitrateChanged(uint32_t ssrc, + uint32_t bitrate) = 0; + + virtual ~RtpRemoteBitrateObserver() {} +}; } // namespace webrtc #endif // WEBRTC_MODULES_RTP_RTCP_INTERFACE_RTP_RTCP_DEFINES_H_ diff --git a/src/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h b/src/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h index 2ac912590..1b5e407d0 100644 --- a/src/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h +++ b/src/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h @@ -8,10 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef WEBRTC_MODULES_RTP_RTCP_MOCKS_MOCK_RTP_RTCP_H_ -#define WEBRTC_MODULES_RTP_RTCP_MOCKS_MOCK_RTP_RTCP_H_ - -#include +#include "../testing/gmock/include/gmock/gmock.h" #include "modules/interface/module.h" #include "modules/rtp_rtcp/interface/rtp_rtcp.h" @@ -214,7 +211,7 @@ class MockRtpRtcp : public RtpRtcp { MOCK_METHOD3(SetREMBData, WebRtc_Word32(const WebRtc_UWord32 bitrate, const WebRtc_UWord8 numberOfSSRC, const WebRtc_UWord32* SSRC)); MOCK_METHOD1(SetRemoteBitrateObserver, - bool(RemoteBitrateObserver*)); + bool(RtpRemoteBitrateObserver*)); MOCK_CONST_METHOD0(IJ, bool()); MOCK_METHOD1(SetIJStatus, @@ -288,5 +285,3 @@ class MockRtpRtcp : public RtpRtcp { }; } // namespace webrtc - -#endif // WEBRTC_MODULES_RTP_RTCP_MOCKS_MOCK_RTP_RTCP_H_ diff --git a/src/modules/rtp_rtcp/source/Bitrate.h b/src/modules/rtp_rtcp/source/Bitrate.h index 074476bda..3859aaa3a 100644 --- a/src/modules/rtp_rtcp/source/Bitrate.h +++ b/src/modules/rtp_rtcp/source/Bitrate.h @@ -55,6 +55,32 @@ private: WebRtc_UWord32 _packetCount; }; -} // namespace webrtc +struct DataTimeSizeTuple +{ + DataTimeSizeTuple(WebRtc_UWord32 sizeBytes, WebRtc_Word64 timeCompleteMs) : + _sizeBytes(sizeBytes), + _timeCompleteMs(timeCompleteMs) {} + + WebRtc_UWord32 _sizeBytes; + WebRtc_Word64 _timeCompleteMs; +}; + +class BitRateStats +{ +public: + BitRateStats(); + ~BitRateStats(); + + void Init(); + void Update(WebRtc_UWord32 packetSizeBytes, WebRtc_Word64 nowMs); + WebRtc_UWord32 BitRate(WebRtc_Word64 nowMs); + +private: + void EraseOld(WebRtc_Word64 nowMs); + + std::list _dataSamples; + WebRtc_UWord32 _accumulatedBytes; +}; +} // namespace webrtc #endif // WEBRTC_MODULES_RTP_RTCP_SOURCE_BITRATE_H_ diff --git a/src/modules/rtp_rtcp/source/bitrate.cc b/src/modules/rtp_rtcp/source/bitrate.cc index 8f0903400..0fbb7ad3b 100644 --- a/src/modules/rtp_rtcp/source/bitrate.cc +++ b/src/modules/rtp_rtcp/source/bitrate.cc @@ -11,6 +11,8 @@ #include "Bitrate.h" #include "rtp_utility.h" +#define BITRATE_AVERAGE_WINDOW 2000 + namespace webrtc { Bitrate::Bitrate(RtpRtcpClock* clock) : _clock(*clock), @@ -110,4 +112,77 @@ Bitrate::Process() } } +BitRateStats::BitRateStats() + :_dataSamples(), _accumulatedBytes(0) +{ +} + +BitRateStats::~BitRateStats() +{ + while (_dataSamples.size() > 0) + { + delete _dataSamples.front(); + _dataSamples.pop_front(); + } +} + +void BitRateStats::Init() +{ + _accumulatedBytes = 0; + while (_dataSamples.size() > 0) + { + delete _dataSamples.front(); + _dataSamples.pop_front(); + } +} + +void BitRateStats::Update(WebRtc_UWord32 packetSizeBytes, WebRtc_Word64 nowMs) +{ + // Find an empty slot for storing the new sample and at the same time + // accumulate the history. + _dataSamples.push_back(new DataTimeSizeTuple(packetSizeBytes, nowMs)); + _accumulatedBytes += packetSizeBytes; + EraseOld(nowMs); +} + +void BitRateStats::EraseOld(WebRtc_Word64 nowMs) +{ + while (_dataSamples.size() > 0) + { + if (nowMs - _dataSamples.front()->_timeCompleteMs > + BITRATE_AVERAGE_WINDOW) + { + // Delete old sample + _accumulatedBytes -= _dataSamples.front()->_sizeBytes; + delete _dataSamples.front(); + _dataSamples.pop_front(); + } + else + { + break; + } + } +} + +WebRtc_UWord32 BitRateStats::BitRate(WebRtc_Word64 nowMs) +{ + // Calculate the average bit rate the past BITRATE_AVERAGE_WINDOW ms. + // Removes any old samples from the list. + EraseOld(nowMs); + WebRtc_Word64 timeOldest = nowMs; + if (_dataSamples.size() > 0) + { + timeOldest = _dataSamples.front()->_timeCompleteMs; + } + // Update average bit rate + float denom = static_cast(nowMs - timeOldest); + if (nowMs == timeOldest) + { + // Calculate with a one second window when we haven't + // received more than one packet. + denom = 1000.0; + } + return static_cast(_accumulatedBytes * 8.0f * 1000.0f / + denom + 0.5f); +} } // namespace webrtc diff --git a/src/modules/rtp_rtcp/source/bwe_defines.h b/src/modules/rtp_rtcp/source/bwe_defines.h new file mode 100644 index 000000000..8fdc985c8 --- /dev/null +++ b/src/modules/rtp_rtcp/source/bwe_defines.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_RTP_RTCP_SOURCE_BWE_DEFINES_H_ +#define WEBRTC_MODULES_RTP_RTCP_SOURCE_BWE_DEFINES_H_ + +#include "typedefs.h" + +#define BWE_MAX(a,b) ((a)>(b)?(a):(b)) +#define BWE_MIN(a,b) ((a)<(b)?(a):(b)) + +namespace webrtc { +enum BandwidthUsage +{ + kBwNormal, + kBwOverusing, + kBwUnderUsing +}; + +enum RateControlState +{ + kRcHold, + kRcIncrease, + kRcDecrease +}; + +enum RateControlRegion +{ + kRcNearMax, + kRcAboveMax, + kRcMaxUnknown +}; + +class RateControlInput +{ +public: + RateControlInput(BandwidthUsage bwState, + WebRtc_UWord32 incomingBitRate, + double noiseVar) : + _bwState(bwState), _incomingBitRate(incomingBitRate), _noiseVar(noiseVar) + {}; + + BandwidthUsage _bwState; + WebRtc_UWord32 _incomingBitRate; + double _noiseVar; +}; +} //namespace webrtc + +#endif // WEBRTC_MODULES_RTP_RTCP_SOURCE_BWE_DEFINES_H_ diff --git a/src/modules/rtp_rtcp/source/mock/mock_rtp_receiver_video.h b/src/modules/rtp_rtcp/source/mock/mock_rtp_receiver_video.h index 9498b74be..cb7ebbae5 100644 --- a/src/modules/rtp_rtcp/source/mock/mock_rtp_receiver_video.h +++ b/src/modules/rtp_rtcp/source/mock/mock_rtp_receiver_video.h @@ -8,16 +8,12 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef WEBRTC_MODULES_RTP_RTCP_SOURCE_MOCK_MOCK_RTP_RECEIVER_VIDEO_H_ -#define WEBRTC_MODULES_RTP_RTCP_SOURCE_MOCK_MOCK_RTP_RECEIVER_VIDEO_H_ - #include "modules/rtp_rtcp/source/rtp_receiver_video.h" namespace webrtc { class MockRTPReceiverVideo : public RTPReceiverVideo { public: - MockRTPReceiverVideo() : RTPReceiverVideo(0, NULL, NULL) {} MOCK_METHOD1(ChangeUniqueId, void(const WebRtc_Word32 id)); MOCK_METHOD3(ReceiveRecoveredPacketCallback, @@ -43,5 +39,3 @@ class MockRTPReceiverVideo : public RTPReceiverVideo { }; } // namespace webrtc - -#endif //WEBRTC_MODULES_RTP_RTCP_SOURCE_MOCK_MOCK_RTP_RECEIVER_VIDEO_H_ diff --git a/src/modules/rtp_rtcp/source/overuse_detector.cc b/src/modules/rtp_rtcp/source/overuse_detector.cc new file mode 100644 index 000000000..be609eeca --- /dev/null +++ b/src/modules/rtp_rtcp/source/overuse_detector.cc @@ -0,0 +1,423 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include // abs +#if _WIN32 +#include +#endif + +#include "modules/rtp_rtcp/source/overuse_detector.h" +#include "modules/rtp_rtcp/source/remote_rate_control.h" +#include "modules/rtp_rtcp/source/rtp_utility.h" +#include "system_wrappers/interface/trace.h" + +#ifdef WEBRTC_BWE_MATLAB +extern MatlabEngine eng; // global variable defined elsewhere +#endif + +#define INIT_CAPACITY_SLOPE 8.0/512.0 +#define DETECTOR_THRESHOLD 25.0 +#define OVER_USING_TIME_THRESHOLD 100 +#define MIN_FRAME_PERIOD_HISTORY_LEN 60 + +namespace webrtc { +OverUseDetector::OverUseDetector() + : firstPacket_(true), + currentFrame_(), + prevFrame_(), + numOfDeltas_(0), + slope_(INIT_CAPACITY_SLOPE), + offset_(0), + E_(), + processNoise_(), + avgNoise_(0.0), + varNoise_(500), + threshold_(DETECTOR_THRESHOLD), + tsDeltaHist_(), + prevOffset_(0.0), + timeOverUsing_(-1), + overUseCounter_(0), +#ifndef WEBRTC_BWE_MATLAB + hypothesis_(kBwNormal) { +#else + plot1_(NULL), + plot2_(NULL), + plot3_(NULL), + plot4_(NULL) { +#endif + E_[0][0] = 100; + E_[1][1] = 1e-1; + E_[0][1] = E_[1][0] = 0; + processNoise_[0] = 1e-10; + processNoise_[1] = 1e-2; +} + +OverUseDetector::~OverUseDetector() { +#ifdef WEBRTC_BWE_MATLAB + if (plot1_) { + eng.DeletePlot(plot1_); + plot1_ = NULL; + } + if (plot2_) { + eng.DeletePlot(plot2_); + plot2_ = NULL; + } + if (plot3_) { + eng.DeletePlot(plot3_); + plot3_ = NULL; + } + if (plot4_) { + eng.DeletePlot(plot4_); + plot4_ = NULL; + } +#endif + + tsDeltaHist_.clear(); +} + +void OverUseDetector::Reset() { + firstPacket_ = true; + currentFrame_.size_ = 0; + currentFrame_.completeTimeMs_ = -1; + currentFrame_.timestamp_ = -1; + prevFrame_.size_ = 0; + prevFrame_.completeTimeMs_ = -1; + prevFrame_.timestamp_ = -1; + numOfDeltas_ = 0; + slope_ = INIT_CAPACITY_SLOPE; + offset_ = 0; + E_[0][0] = 100; + E_[1][1] = 1e-1; + E_[0][1] = E_[1][0] = 0; + processNoise_[0] = 1e-10; + processNoise_[1] = 1e-2; + avgNoise_ = 0.0; + varNoise_ = 500; + threshold_ = DETECTOR_THRESHOLD; + prevOffset_ = 0.0; + timeOverUsing_ = -1; + overUseCounter_ = 0; + hypothesis_ = kBwNormal; + tsDeltaHist_.clear(); +} + +bool OverUseDetector::Update(const WebRtcRTPHeader& rtpHeader, + const WebRtc_UWord16 packetSize, + const WebRtc_Word64 nowMS) { +#ifdef WEBRTC_BWE_MATLAB + // Create plots + const WebRtc_Word64 startTimeMs = nowMS; + if (plot1_ == NULL) { + plot1_ = eng.NewPlot(new MatlabPlot()); + plot1_->AddLine(1000, "b.", "scatter"); + } + if (plot2_ == NULL) { + plot2_ = eng.NewPlot(new MatlabPlot()); + plot2_->AddTimeLine(30, "b", "offset", startTimeMs); + plot2_->AddTimeLine(30, "r--", "limitPos", startTimeMs); + plot2_->AddTimeLine(30, "k.", "trigger", startTimeMs); + plot2_->AddTimeLine(30, "ko", "detection", startTimeMs); + // plot2_->AddTimeLine(30, "g", "slowMean", startTimeMs); + } + if (plot3_ == NULL) { + plot3_ = eng.NewPlot(new MatlabPlot()); + plot3_->AddTimeLine(30, "b", "noiseVar", startTimeMs); + } + if (plot4_ == NULL) { + plot4_ = eng.NewPlot(new MatlabPlot()); + // plot4_->AddTimeLine(60, "b", "p11", startTimeMs); + // plot4_->AddTimeLine(60, "r", "p12", startTimeMs); + plot4_->AddTimeLine(60, "g", "p22", startTimeMs); + // plot4_->AddTimeLine(60, "g--", "p22_hat", startTimeMs); + // plot4_->AddTimeLine(30, "b.-", "deltaFs", startTimeMs); + } + +#endif + + bool wrapped = false; + bool completeFrame = false; + if (currentFrame_.timestamp_ == -1) { + currentFrame_.timestamp_ = rtpHeader.header.timestamp; + } else if (ModuleRTPUtility::OldTimestamp( + rtpHeader.header.timestamp, + static_cast(currentFrame_.timestamp_), + &wrapped)) { + // Don't update with old data + return completeFrame; + } else if (rtpHeader.header.timestamp != currentFrame_.timestamp_) { + // First packet of a later frame, the previous frame sample is ready + WEBRTC_TRACE(kTraceStream, kTraceRtpRtcp, -1, + "Frame complete at %I64i", currentFrame_.completeTimeMs_); + if (prevFrame_.completeTimeMs_ >= 0) { // This is our second frame + WebRtc_Word64 tDelta = 0; + double tsDelta = 0; + // Check for wrap + ModuleRTPUtility::OldTimestamp( + static_cast(prevFrame_.timestamp_), + static_cast(currentFrame_.timestamp_), + &wrapped); + CompensatedTimeDelta(currentFrame_, prevFrame_, tDelta, tsDelta, + wrapped); + UpdateKalman(tDelta, tsDelta, currentFrame_.size_, + prevFrame_.size_); + } + // The new timestamp is now the current frame, + // and the old timestamp becomes the previous frame. + prevFrame_ = currentFrame_; + currentFrame_.timestamp_ = rtpHeader.header.timestamp; + currentFrame_.size_ = 0; + currentFrame_.completeTimeMs_ = -1; + completeFrame = true; + } + // Accumulate the frame size + currentFrame_.size_ += packetSize; + currentFrame_.completeTimeMs_ = nowMS; + return completeFrame; +} + +BandwidthUsage OverUseDetector::State() const { + return hypothesis_; +} + +double OverUseDetector::NoiseVar() const { + return varNoise_; +} + +void OverUseDetector::SetRateControlRegion(RateControlRegion region) { + switch (region) { + case kRcMaxUnknown: { + threshold_ = DETECTOR_THRESHOLD; + break; + } + case kRcAboveMax: + case kRcNearMax: { + threshold_ = DETECTOR_THRESHOLD / 2; + break; + } + } +} + +void OverUseDetector::CompensatedTimeDelta(const FrameSample& currentFrame, + const FrameSample& prevFrame, + WebRtc_Word64& tDelta, + double& tsDelta, + bool wrapped) { + numOfDeltas_++; + if (numOfDeltas_ > 1000) { + numOfDeltas_ = 1000; + } + // Add wrap-around compensation + WebRtc_Word64 wrapCompensation = 0; + if (wrapped) { + wrapCompensation = static_cast(1)<<32; + } + tsDelta = (currentFrame.timestamp_ + + wrapCompensation + - prevFrame.timestamp_) / 90.0; + tDelta = currentFrame.completeTimeMs_ - prevFrame.completeTimeMs_; + assert(tsDelta > 0); +} + +double OverUseDetector::CurrentDrift() { + return 1.0; +} + +void OverUseDetector::UpdateKalman(WebRtc_Word64 tDelta, + double tsDelta, + WebRtc_UWord32 frameSize, + WebRtc_UWord32 prevFrameSize) { + const double minFramePeriod = UpdateMinFramePeriod(tsDelta); + const double drift = CurrentDrift(); + // Compensate for drift + const double tTsDelta = tDelta - tsDelta / drift; + double fsDelta = static_cast(frameSize) - prevFrameSize; + + // Update the Kalman filter + const double scaleFactor = minFramePeriod / (1000.0 / 30.0); + E_[0][0] += processNoise_[0] * scaleFactor; + E_[1][1] += processNoise_[1] * scaleFactor; + + if ((hypothesis_ == kBwOverusing && offset_ < prevOffset_) || + (hypothesis_ == kBwUnderUsing && offset_ > prevOffset_)) { + E_[1][1] += 10 * processNoise_[1] * scaleFactor; + } + + const double h[2] = {fsDelta, 1.0}; + const double Eh[2] = {E_[0][0]*h[0] + E_[0][1]*h[1], + E_[1][0]*h[0] + E_[1][1]*h[1]}; + + const double residual = tTsDelta - slope_*h[0] - offset_; + + const bool stableState = + (BWE_MIN(numOfDeltas_, 60) * abs(offset_) < threshold_); + // We try to filter out very late frames. For instance periodic key + // frames doesn't fit the Gaussian model well. + if (abs(residual) < 3 * sqrt(varNoise_)) { + UpdateNoiseEstimate(residual, minFramePeriod, stableState); + } else { + UpdateNoiseEstimate(3 * sqrt(varNoise_), minFramePeriod, stableState); + } + + const double denom = varNoise_ + h[0]*Eh[0] + h[1]*Eh[1]; + + const double K[2] = {Eh[0] / denom, + Eh[1] / denom}; + + const double IKh[2][2] = {{1.0 - K[0]*h[0], -K[0]*h[1]}, + {-K[1]*h[0], 1.0 - K[1]*h[1]}}; + const double e00 = E_[0][0]; + const double e01 = E_[0][1]; + + // Update state + E_[0][0] = e00 * IKh[0][0] + E_[1][0] * IKh[0][1]; + E_[0][1] = e01 * IKh[0][0] + E_[1][1] * IKh[0][1]; + E_[1][0] = e00 * IKh[1][0] + E_[1][0] * IKh[1][1]; + E_[1][1] = e01 * IKh[1][0] + E_[1][1] * IKh[1][1]; + + // Covariance matrix, must be positive semi-definite + assert(E_[0][0] + E_[1][1] >= 0 && + E_[0][0] * E_[1][1] - E_[0][1] * E_[1][0] >= 0 && + E_[0][0] >= 0); + +#ifdef WEBRTC_BWE_MATLAB + // plot4_->Append("p11",E_[0][0]); + // plot4_->Append("p12",E_[0][1]); + plot4_->Append("p22", E_[1][1]); + // plot4_->Append("p22_hat", 0.5*(processNoise_[1] + + // sqrt(processNoise_[1]*(processNoise_[1] + 4*varNoise_)))); + // plot4_->Append("deltaFs", fsDelta); + plot4_->Plot(); +#endif + slope_ = slope_ + K[0] * residual; + prevOffset_ = offset_; + offset_ = offset_ + K[1] * residual; + + Detect(tsDelta); + +#ifdef WEBRTC_BWE_MATLAB + plot1_->Append("scatter", + static_cast(currentFrame_.size_) - prevFrame_.size_, + static_cast(tDelta-tsDelta)); + plot1_->MakeTrend("scatter", "slope", slope_, offset_, "k-"); + plot1_->MakeTrend("scatter", "thresholdPos", + slope_, offset_ + 2 * sqrt(varNoise_), "r-"); + plot1_->MakeTrend("scatter", "thresholdNeg", + slope_, offset_ - 2 * sqrt(varNoise_), "r-"); + plot1_->Plot(); + + plot2_->Append("offset", offset_); + plot2_->Append("limitPos", threshold_/BWE_MIN(numOfDeltas_, 60)); + plot2_->Plot(); + + plot3_->Append("noiseVar", varNoise_); + plot3_->Plot(); +#endif +} + +double OverUseDetector::UpdateMinFramePeriod(double tsDelta) { + double minFramePeriod = tsDelta; + if (tsDeltaHist_.size() >= MIN_FRAME_PERIOD_HISTORY_LEN) { + std::list::iterator firstItem = tsDeltaHist_.begin(); + tsDeltaHist_.erase(firstItem); + } + std::list::iterator it = tsDeltaHist_.begin(); + for (; it != tsDeltaHist_.end(); it++) { + minFramePeriod = BWE_MIN(*it, minFramePeriod); + } + tsDeltaHist_.push_back(tsDelta); + return minFramePeriod; +} + +void OverUseDetector::UpdateNoiseEstimate(double residual, + double tsDelta, + bool stableState) { + if (!stableState) { + return; + } + // Faster filter during startup to faster adapt to the jitter level + // of the network alpha is tuned for 30 frames per second, but + double alpha = 0.01; + if (numOfDeltas_ > 10*30) { + alpha = 0.002; + } + // Only update the noise estimate if we're not over-using + // beta is a function of alpha and the time delta since + // the previous update. + const double beta = pow(1 - alpha, tsDelta * 30.0 / 1000.0); + avgNoise_ = beta * avgNoise_ + + (1 - beta) * residual; + varNoise_ = beta * varNoise_ + + (1 - beta) * (avgNoise_ - residual) * (avgNoise_ - residual); + if (varNoise_ < 1e-7) { + varNoise_ = 1e-7; + } +} + +BandwidthUsage OverUseDetector::Detect(double tsDelta) { + if (numOfDeltas_ < 2) { + return kBwNormal; + } + const double T = BWE_MIN(numOfDeltas_, 60) * offset_; + if (abs(T) > threshold_) { + if (offset_ > 0) { + if (timeOverUsing_ == -1) { + // Initialize the timer. Assume that we've been + // over-using half of the time since the previous + // sample. + timeOverUsing_ = tsDelta / 2; + } else { + // Increment timer + timeOverUsing_ += tsDelta; + } + overUseCounter_++; + if (timeOverUsing_ > OVER_USING_TIME_THRESHOLD + && overUseCounter_ > 1) { + if (offset_ >= prevOffset_) { +#ifdef _DEBUG + if (hypothesis_ != kBwOverusing) { + WEBRTC_TRACE(kTraceStream, kTraceRtpRtcp, -1, "BWE: kBwOverusing"); + } +#endif + timeOverUsing_ = 0; + overUseCounter_ = 0; + hypothesis_ = kBwOverusing; +#ifdef WEBRTC_BWE_MATLAB + plot2_->Append("detection", offset_); // plot it later +#endif + } + } +#ifdef WEBRTC_BWE_MATLAB + plot2_->Append("trigger", offset_); // plot it later +#endif + } else { +#ifdef _DEBUG + if (hypothesis_ != kBwUnderUsing) { + WEBRTC_TRACE(kTraceStream, kTraceRtpRtcp, -1, "BWE: kBwUnderUsing"); + } +#endif + timeOverUsing_ = -1; + overUseCounter_ = 0; + hypothesis_ = kBwUnderUsing; + } + } else { +#ifdef _DEBUG + if (hypothesis_ != kBwNormal) { + WEBRTC_TRACE(kTraceStream, kTraceRtpRtcp, -1, "BWE: kBwNormal"); + } +#endif + timeOverUsing_ = -1; + overUseCounter_ = 0; + hypothesis_ = kBwNormal; + } + return hypothesis_; +} + +} // namespace webrtc diff --git a/src/modules/rtp_rtcp/source/overuse_detector.h b/src/modules/rtp_rtcp/source/overuse_detector.h new file mode 100644 index 000000000..3b432fb0f --- /dev/null +++ b/src/modules/rtp_rtcp/source/overuse_detector.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef WEBRTC_MODULES_RTP_RTCP_SOURCE_OVERUSE_DETECTOR_H_ +#define WEBRTC_MODULES_RTP_RTCP_SOURCE_OVERUSE_DETECTOR_H_ + +#include + +#include "modules/interface/module_common_types.h" +#include "modules/rtp_rtcp/source/bwe_defines.h" +#include "typedefs.h" // NOLINT(build/include) + +#ifdef WEBRTC_BWE_MATLAB +#include "../test/BWEStandAlone/MatlabPlot.h" +#endif + +namespace webrtc { +enum RateControlRegion; + +class OverUseDetector { + public: + OverUseDetector(); + ~OverUseDetector(); + bool Update(const WebRtcRTPHeader& rtpHeader, + const WebRtc_UWord16 packetSize, + const WebRtc_Word64 nowMS); + BandwidthUsage State() const; + void Reset(); + double NoiseVar() const; + void SetRateControlRegion(RateControlRegion region); + + private: + struct FrameSample { + FrameSample() : size_(0), completeTimeMs_(-1), timestamp_(-1) {} + + WebRtc_UWord32 size_; + WebRtc_Word64 completeTimeMs_; + WebRtc_Word64 timestamp_; + }; + + void CompensatedTimeDelta(const FrameSample& currentFrame, + const FrameSample& prevFrame, + WebRtc_Word64& tDelta, + double& tsDelta, + bool wrapped); + void UpdateKalman(WebRtc_Word64 tDelta, + double tsDelta, + WebRtc_UWord32 frameSize, + WebRtc_UWord32 prevFrameSize); + double UpdateMinFramePeriod(double tsDelta); + void UpdateNoiseEstimate(double residual, double tsDelta, bool stableState); + BandwidthUsage Detect(double tsDelta); + double CurrentDrift(); + + bool firstPacket_; + FrameSample currentFrame_; + FrameSample prevFrame_; + WebRtc_UWord16 numOfDeltas_; + double slope_; + double offset_; + double E_[2][2]; + double processNoise_[2]; + double avgNoise_; + double varNoise_; + double threshold_; + std::list tsDeltaHist_; + double prevOffset_; + double timeOverUsing_; + WebRtc_UWord16 overUseCounter_; + BandwidthUsage hypothesis_; + +#ifdef WEBRTC_BWE_MATLAB + MatlabPlot* plot1_; + MatlabPlot* plot2_; + MatlabPlot* plot3_; + MatlabPlot* plot4_; +#endif +}; +} // namespace webrtc + +#endif // WEBRTC_MODULES_RTP_RTCP_SOURCE_OVERUSE_DETECTOR_H_ diff --git a/src/modules/rtp_rtcp/source/remote_rate_control.cc b/src/modules/rtp_rtcp/source/remote_rate_control.cc new file mode 100644 index 000000000..fe8477c6b --- /dev/null +++ b/src/modules/rtp_rtcp/source/remote_rate_control.cc @@ -0,0 +1,483 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#if _WIN32 +#include +#endif + +#include "remote_rate_control.h" +#include "trace.h" +#include +#include + +#ifdef MATLAB +extern MatlabEngine eng; // global variable defined elsewhere +#endif + +namespace webrtc { +RemoteRateControl::RemoteRateControl() +: +_minConfiguredBitRate(30000), +_maxConfiguredBitRate(30000000), +_currentBitRate(_maxConfiguredBitRate), +_maxHoldRate(0), +_avgMaxBitRate(-1.0f), +_varMaxBitRate(0.4f), +_rcState(kRcHold), +_cameFromState(kRcDecrease), +_rcRegion(kRcMaxUnknown), +_lastBitRateChange(-1), +_currentInput(kBwNormal, 0, 1.0), +_updated(false), +_timeFirstIncomingEstimate(-1), +_initializedBitRate(false), +_avgChangePeriod(1000.0f), +_lastChangeMs(-1), +_beta(0.9f) +#ifdef MATLAB +,_plot1(NULL), +_plot2(NULL) +#endif +{ +} + +RemoteRateControl::~RemoteRateControl() +{ +#ifdef MATLAB + eng.DeletePlot(_plot1); + eng.DeletePlot(_plot2); +#endif +} + +void RemoteRateControl::Reset() +{ + _minConfiguredBitRate = 30000; + _maxConfiguredBitRate = 30000000; + _currentBitRate = _maxConfiguredBitRate; + _maxHoldRate = 0; + _avgMaxBitRate = -1.0f; + _varMaxBitRate = 0.4f; + _rcState = kRcHold; + _cameFromState = kRcHold; + _rcRegion = kRcMaxUnknown; + _lastBitRateChange = -1; + _avgChangePeriod = 1000.0f; + _lastChangeMs = -1; + _beta = 0.9f; + _currentInput._bwState = kBwNormal; + _currentInput._incomingBitRate = 0; + _currentInput._noiseVar = 1.0; + _updated = false; + _timeFirstIncomingEstimate = -1; + _initializedBitRate = false; +} + +bool RemoteRateControl::ValidEstimate() const { + return _initializedBitRate; +} + +WebRtc_Word32 RemoteRateControl::SetConfiguredBitRates(WebRtc_UWord32 minBitRateBps, WebRtc_UWord32 maxBitRateBps) +{ + if (minBitRateBps > maxBitRateBps) + { + return -1; + } + _minConfiguredBitRate = minBitRateBps; + _maxConfiguredBitRate = maxBitRateBps; + _currentBitRate = BWE_MIN(BWE_MAX(minBitRateBps, _currentBitRate), maxBitRateBps); + return 0; +} + +WebRtc_UWord32 RemoteRateControl::LatestEstimate() const { + return _currentBitRate; +} + +WebRtc_UWord32 RemoteRateControl::UpdateBandwidthEstimate(WebRtc_UWord32 RTT, + WebRtc_Word64 nowMS) +{ + _currentBitRate = ChangeBitRate(_currentBitRate, _currentInput._incomingBitRate, + _currentInput._noiseVar, RTT, nowMS); + return _currentBitRate; +} + +RateControlRegion RemoteRateControl::Update(const RateControlInput& input, + bool& firstOverUse, + WebRtc_Word64 nowMS) +{ +#ifdef MATLAB + // Create plots + if (_plot1 == NULL) + { + _plot1 = eng.NewPlot(new MatlabPlot()); + + _plot1->AddTimeLine(30, "b", "current"); + _plot1->AddTimeLine(30, "r-", "avgMax"); + _plot1->AddTimeLine(30, "r--", "pStdMax"); + _plot1->AddTimeLine(30, "r--", "nStdMax"); + _plot1->AddTimeLine(30, "r+", "max"); + _plot1->AddTimeLine(30, "g", "incoming"); + _plot1->AddTimeLine(30, "b+", "recovery"); + } + if (_plot2 == NULL) + { + _plot2 = eng.NewPlot(new MatlabPlot()); + + _plot2->AddTimeLine(30, "b", "alpha"); + } +#endif + + firstOverUse = (_currentInput._bwState != kBwOverusing && + input._bwState == kBwOverusing); + + // Set the initial bit rate value to what we're receiving the first second + if (!_initializedBitRate) + { + if (_timeFirstIncomingEstimate < 0) + { + if (input._incomingBitRate > 0) + { + _timeFirstIncomingEstimate = nowMS; + } + } + else if (nowMS - _timeFirstIncomingEstimate > 1000 && + input._incomingBitRate > 0) + { + _currentBitRate = input._incomingBitRate; + _initializedBitRate = true; + } + } + + if (_updated && _currentInput._bwState == kBwOverusing) + { + // Only update delay factor and incoming bit rate. We always want to react on an over-use. + _currentInput._noiseVar = input._noiseVar; + _currentInput._incomingBitRate = input._incomingBitRate; + return _rcRegion; + } + _updated = true; + _currentInput = input; + WEBRTC_TRACE(kTraceStream, kTraceRtpRtcp, -1, "BWE: Incoming rate = %u kbps", input._incomingBitRate/1000); + return _rcRegion; +} + +WebRtc_UWord32 RemoteRateControl::ChangeBitRate(WebRtc_UWord32 currentBitRate, + WebRtc_UWord32 incomingBitRate, + double noiseVar, + WebRtc_UWord32 RTT, + WebRtc_Word64 nowMS) +{ + if (!_updated) + { + return _currentBitRate; + } + _updated = false; + UpdateChangePeriod(nowMS); + ChangeState(_currentInput, nowMS); + // calculated here because it's used in multiple places + const float incomingBitRateKbps = incomingBitRate / 1000.0f; + // Calculate the max bit rate std dev given the normalized + // variance and the current incoming bit rate. + const float stdMaxBitRate = sqrt(_varMaxBitRate * _avgMaxBitRate); + bool recovery = false; + switch (_rcState) + { + case kRcHold: + { + _maxHoldRate = BWE_MAX(_maxHoldRate, incomingBitRate); + break; + } + case kRcIncrease: + { + if (_avgMaxBitRate >= 0) + { + if (incomingBitRateKbps > _avgMaxBitRate + 3 * stdMaxBitRate) + { + ChangeRegion(kRcMaxUnknown); + _avgMaxBitRate = -1.0; + } + else if (incomingBitRateKbps > _avgMaxBitRate + 2.5 * stdMaxBitRate) + { + ChangeRegion(kRcAboveMax); + } + } + WEBRTC_TRACE(kTraceStream, kTraceRtpRtcp, -1, + "BWE: Response time: %f + %i + 10*33\n", + _avgChangePeriod, RTT); + const WebRtc_UWord32 responseTime = static_cast(_avgChangePeriod + 0.5f) + RTT + 300; + double alpha = RateIncreaseFactor(nowMS, _lastBitRateChange, + responseTime, noiseVar); + + WEBRTC_TRACE(kTraceStream, kTraceRtpRtcp, -1, + "BWE: _avgChangePeriod = %f ms; RTT = %u ms", _avgChangePeriod, RTT); + + currentBitRate = static_cast(currentBitRate * alpha) + 1000; + if (_maxHoldRate > 0 && _beta * _maxHoldRate > currentBitRate) + { + currentBitRate = static_cast(_beta * _maxHoldRate); + _avgMaxBitRate = _beta * _maxHoldRate / 1000.0f; + ChangeRegion(kRcNearMax); + recovery = true; +#ifdef MATLAB + _plot1->Append("recovery", _maxHoldRate/1000); +#endif + } + _maxHoldRate = 0; + WEBRTC_TRACE(kTraceStream, kTraceRtpRtcp, -1, + "BWE: Increase rate to currentBitRate = %u kbps", currentBitRate/1000); + _lastBitRateChange = nowMS; + break; + } + case kRcDecrease: + { + if (incomingBitRate < _minConfiguredBitRate) + { + currentBitRate = _minConfiguredBitRate; + } + else + { + // Set bit rate to something slightly lower than max + // to get rid of any self-induced delay. + currentBitRate = static_cast(_beta * incomingBitRate + 0.5); + if (currentBitRate > _currentBitRate) + { + // Avoid increasing the rate when over-using. + if (_rcRegion != kRcMaxUnknown) + { + currentBitRate = static_cast(_beta * _avgMaxBitRate * 1000 + 0.5f); + } + currentBitRate = BWE_MIN(currentBitRate, _currentBitRate); + } + ChangeRegion(kRcNearMax); + + if (incomingBitRateKbps < _avgMaxBitRate - 3 * stdMaxBitRate) + { + _avgMaxBitRate = -1.0f; + } + + UpdateMaxBitRateEstimate(incomingBitRateKbps); + +#ifdef MATLAB + _plot1->Append("max", incomingBitRateKbps); +#endif + + WEBRTC_TRACE(kTraceStream, kTraceRtpRtcp, -1, "BWE: Decrease rate to currentBitRate = %u kbps", currentBitRate/1000); + } + // Stay on hold until the pipes are cleared. + ChangeState(kRcHold); + _lastBitRateChange = nowMS; + break; + } + } + if (!recovery && (incomingBitRate > 100000 || currentBitRate > 150000) && + currentBitRate > 1.5 * incomingBitRate) + { + // Allow changing the bit rate if we are operating at very low rates + // Don't change the bit rate if the send side is too far off + currentBitRate = _currentBitRate; + _lastBitRateChange = nowMS; + } +#ifdef MATLAB + if (_avgMaxBitRate >= 0.0f) + { + _plot1->Append("avgMax", _avgMaxBitRate); + _plot1->Append("pStdMax", _avgMaxBitRate + 3*stdMaxBitRate); + _plot1->Append("nStdMax", _avgMaxBitRate - 3*stdMaxBitRate); + } + _plot1->Append("incoming", incomingBitRate/1000); + _plot1->Append("current", currentBitRate/1000); + _plot1->Plot(); +#endif + return currentBitRate; +} + +double RemoteRateControl::RateIncreaseFactor(WebRtc_Word64 nowMs, WebRtc_Word64 lastMs, WebRtc_UWord32 reactionTimeMs, double noiseVar) const +{ + // alpha = 1.02 + B ./ (1 + exp(b*(tr - (c1*s2 + c2)))) + // Parameters + const double B = 0.0407; + const double b = 0.0025; + const double c1 = -6700.0 / (33 * 33); + const double c2 = 800.0; + const double d = 0.85; + + double alpha = 1.005 + B / (1 + exp( b * (d * reactionTimeMs - (c1 * noiseVar + c2)))); + + if (alpha < 1.005) + { + alpha = 1.005; + } + else if (alpha > 1.3) + { + alpha = 1.3; + } + + WEBRTC_TRACE(kTraceStream, kTraceRtpRtcp, -1, + "BWE: alpha = %f", alpha); +#ifdef MATLAB + _plot2->Append("alpha", alpha); + _plot2->Plot(); +#endif + + if (lastMs > -1) + { + alpha = pow(alpha, (nowMs - lastMs) / 1000.0); + } + + if (_rcRegion == kRcNearMax) + { + // We're close to our previous maximum. Try to stabilize the + // bit rate in this region, by increasing in smaller steps. + alpha = alpha - (alpha - 1.0) / 2.0; + } + else if (_rcRegion == kRcMaxUnknown) + { + alpha = alpha + (alpha - 1.0) * 2.0; + } + + return alpha; +} + +void RemoteRateControl::UpdateChangePeriod(WebRtc_Word64 nowMs) +{ + WebRtc_Word64 changePeriod = 0; + if (_lastChangeMs > -1) + { + changePeriod = nowMs - _lastChangeMs; + } + _lastChangeMs = nowMs; + _avgChangePeriod = 0.9f * _avgChangePeriod + 0.1f * changePeriod; +} + +void RemoteRateControl::UpdateMaxBitRateEstimate(float incomingBitRateKbps) +{ + const float alpha = 0.05f; + if (_avgMaxBitRate == -1.0f) + { + _avgMaxBitRate = incomingBitRateKbps; + } + else + { + _avgMaxBitRate = (1 - alpha) * _avgMaxBitRate + + alpha * incomingBitRateKbps; + } + // Estimate the max bit rate variance and normalize the variance + // with the average max bit rate. + const float norm = BWE_MAX(_avgMaxBitRate, 1.0f); + _varMaxBitRate = (1 - alpha) * _varMaxBitRate + + alpha * (_avgMaxBitRate - incomingBitRateKbps) * + (_avgMaxBitRate - incomingBitRateKbps) / + norm; + // 0.4 ~= 14 kbit/s at 500 kbit/s + if (_varMaxBitRate < 0.4f) + { + _varMaxBitRate = 0.4f; + } + // 2.5f ~= 35 kbit/s at 500 kbit/s + if (_varMaxBitRate > 2.5f) + { + _varMaxBitRate = 2.5f; + } +} + +void RemoteRateControl::ChangeState(const RateControlInput& input, WebRtc_Word64 nowMs) +{ + switch (_currentInput._bwState) + { + case kBwNormal: + { + if (_rcState == kRcHold) + { + _lastBitRateChange = nowMs; + ChangeState(kRcIncrease); + } + break; + } + case kBwOverusing: + { + if (_rcState != kRcDecrease) + { + ChangeState(kRcDecrease); + } + break; + } + case kBwUnderUsing: + { + ChangeState(kRcHold); + break; + } + } +} + +void RemoteRateControl::ChangeRegion(RateControlRegion region) +{ + _rcRegion = region; + switch (_rcRegion) + { + case kRcAboveMax: + case kRcMaxUnknown: + { + _beta = 0.9f; + break; + } + case kRcNearMax: + { + _beta = 0.95f; + break; + } + } +} + +void RemoteRateControl::ChangeState(RateControlState newState) +{ + _cameFromState = _rcState; + _rcState = newState; + char state1[15]; + char state2[15]; + char state3[15]; + StateStr(_cameFromState, state1); + StateStr(_rcState, state2); + StateStr(_currentInput._bwState, state3); + WEBRTC_TRACE(kTraceStream, kTraceRtpRtcp, -1, + "\t%s => %s due to %s\n", state1, state2, state3); +} + +void RemoteRateControl::StateStr(RateControlState state, char* str) +{ + switch (state) + { + case kRcDecrease: + strncpy(str, "DECREASE", 9); + break; + case kRcHold: + strncpy(str, "HOLD", 5); + break; + case kRcIncrease: + strncpy(str, "INCREASE", 9); + break; + } +} + +void RemoteRateControl::StateStr(BandwidthUsage state, char* str) +{ + switch (state) + { + case kBwNormal: + strncpy(str, "NORMAL", 7); + break; + case kBwOverusing: + strncpy(str, "OVER USING", 11); + break; + case kBwUnderUsing: + strncpy(str, "UNDER USING", 12); + break; + } +} + +} // namespace webrtc diff --git a/src/modules/rtp_rtcp/source/remote_rate_control.h b/src/modules/rtp_rtcp/source/remote_rate_control.h new file mode 100644 index 000000000..197bf223c --- /dev/null +++ b/src/modules/rtp_rtcp/source/remote_rate_control.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_RTP_RTCP_SOURCE_REMOTE_RATE_CONTROL_H_ +#define WEBRTC_MODULES_RTP_RTCP_SOURCE_REMOTE_RATE_CONTROL_H_ + +#include "bwe_defines.h" +#include "typedefs.h" + +#ifdef MATLAB +#include "../test/BWEStandAlone/MatlabPlot.h" +#endif + +namespace webrtc { +class RemoteRateControl +{ +public: + RemoteRateControl(); + ~RemoteRateControl(); + WebRtc_Word32 SetConfiguredBitRates(WebRtc_UWord32 minBitRate, + WebRtc_UWord32 maxBitRate); + WebRtc_UWord32 LatestEstimate() const; + WebRtc_UWord32 UpdateBandwidthEstimate(WebRtc_UWord32 RTT, + WebRtc_Word64 nowMS); + RateControlRegion Update(const RateControlInput& input, bool& firstOverUse, + WebRtc_Word64 nowMS); + void Reset(); + + // Returns true if there is a valid estimate of the incoming bitrate, false + // otherwise. + bool ValidEstimate() const; + +private: + WebRtc_UWord32 ChangeBitRate(WebRtc_UWord32 currentBitRate, + WebRtc_UWord32 incomingBitRate, + double delayFactor, WebRtc_UWord32 RTT, + WebRtc_Word64 nowMS); + double RateIncreaseFactor(WebRtc_Word64 nowMs, + WebRtc_Word64 lastMs, + WebRtc_UWord32 reactionTimeMs, + double noiseVar) const; + void UpdateChangePeriod(WebRtc_Word64 nowMs); + void UpdateMaxBitRateEstimate(float incomingBitRateKbps); + void ChangeState(const RateControlInput& input, WebRtc_Word64 nowMs); + void ChangeState(RateControlState newState); + void ChangeRegion(RateControlRegion region); + static void StateStr(RateControlState state, char* str); + static void StateStr(BandwidthUsage state, char* str); + + WebRtc_UWord32 _minConfiguredBitRate; + WebRtc_UWord32 _maxConfiguredBitRate; + WebRtc_UWord32 _currentBitRate; + WebRtc_UWord32 _maxHoldRate; + float _avgMaxBitRate; + float _varMaxBitRate; + RateControlState _rcState; + RateControlState _cameFromState; + RateControlRegion _rcRegion; + WebRtc_Word64 _lastBitRateChange; + RateControlInput _currentInput; + bool _updated; + WebRtc_Word64 _timeFirstIncomingEstimate; + bool _initializedBitRate; + + float _avgChangePeriod; + WebRtc_Word64 _lastChangeMs; + float _beta; +#ifdef MATLAB + MatlabPlot *_plot1; + MatlabPlot *_plot2; +#endif +}; +} // namespace webrtc + +#endif // WEBRTC_MODULES_RTP_RTCP_SOURCE_REMOTE_RATE_CONTROL_H_ diff --git a/src/modules/rtp_rtcp/source/rtcp_format_remb_unittest.cc b/src/modules/rtp_rtcp/source/rtcp_format_remb_unittest.cc index 96d1d2f2e..d937f3510 100644 --- a/src/modules/rtp_rtcp/source/rtcp_format_remb_unittest.cc +++ b/src/modules/rtp_rtcp/source/rtcp_format_remb_unittest.cc @@ -16,8 +16,7 @@ #include "rtcp_sender.h" #include "rtcp_receiver.h" #include "rtp_rtcp_impl.h" -#include "modules/remote_bitrate_estimator/include/bwe_defines.h" -#include "modules/remote_bitrate_estimator/include/mock/mock_remote_bitrate_observer.h" +#include "bwe_defines.h" namespace { @@ -58,9 +57,7 @@ class TestTransport : public Transport { class RtcpFormatRembTest : public ::testing::Test { protected: - RtcpFormatRembTest() - : remote_bitrate_observer_(), - remote_bitrate_estimator_(&remote_bitrate_observer_) {} + RtcpFormatRembTest() {}; virtual void SetUp(); virtual void TearDown(); @@ -69,8 +66,6 @@ class RtcpFormatRembTest : public ::testing::Test { RTCPSender* rtcp_sender_; RTCPReceiver* rtcp_receiver_; TestTransport* test_transport_; - MockRemoteBitrateObserver remote_bitrate_observer_; - RemoteBitrateEstimator remote_bitrate_estimator_; }; void RtcpFormatRembTest::SetUp() { @@ -79,7 +74,6 @@ void RtcpFormatRembTest::SetUp() { configuration.id = 0; configuration.audio = false; configuration.clock = system_clock_; - configuration.remote_bitrate_estimator = &remote_bitrate_estimator_; dummy_rtp_rtcp_impl_ = new ModuleRtpRtcpImpl(configuration); rtcp_sender_ = new RTCPSender(0, false, system_clock_, dummy_rtp_rtcp_impl_); rtcp_receiver_ = new RTCPReceiver(0, system_clock_, dummy_rtp_rtcp_impl_); diff --git a/src/modules/rtp_rtcp/source/rtcp_receiver_unittest.cc b/src/modules/rtp_rtcp/source/rtcp_receiver_unittest.cc index b511c4e9c..9598a4331 100644 --- a/src/modules/rtp_rtcp/source/rtcp_receiver_unittest.cc +++ b/src/modules/rtp_rtcp/source/rtcp_receiver_unittest.cc @@ -12,13 +12,10 @@ /* * This file includes unit tests for the RTCPReceiver. */ -#include #include // Note: This file has no directory. Lint warning must be ignored. #include "common_types.h" -#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" -#include "modules/remote_bitrate_estimator/include/mock/mock_remote_bitrate_observer.h" #include "modules/rtp_rtcp/source/rtp_utility.h" #include "modules/rtp_rtcp/source/rtcp_sender.h" #include "modules/rtp_rtcp/source/rtcp_receiver.h" @@ -182,19 +179,15 @@ class TestTransport : public Transport, class RtcpReceiverTest : public ::testing::Test { protected: - RtcpReceiverTest() - : remote_bitrate_observer_(), - remote_bitrate_estimator_(&remote_bitrate_observer_) { + RtcpReceiverTest() { // system_clock_ = ModuleRTPUtility::GetSystemClock(); system_clock_ = new FakeSystemClock(); test_transport_ = new TestTransport(); - RtpRtcp::Configuration configuration; configuration.id = 0; configuration.audio = false; configuration.clock = system_clock_; configuration.outgoing_transport = test_transport_; - configuration.remote_bitrate_estimator = &remote_bitrate_estimator_; rtp_rtcp_impl_ = new ModuleRtpRtcpImpl(configuration); rtcp_receiver_ = new RTCPReceiver(0, system_clock_, rtp_rtcp_impl_); test_transport_->SetRTCPReceiver(rtcp_receiver_); @@ -226,8 +219,6 @@ class RtcpReceiverTest : public ::testing::Test { RTCPReceiver* rtcp_receiver_; TestTransport* test_transport_; RTCPHelp::RTCPPacketInformation rtcp_packet_info_; - MockRemoteBitrateObserver remote_bitrate_observer_; - RemoteBitrateEstimator remote_bitrate_estimator_; }; diff --git a/src/modules/rtp_rtcp/source/rtcp_sender.cc b/src/modules/rtp_rtcp/source/rtcp_sender.cc index eb3a905d9..6383f8f36 100644 --- a/src/modules/rtp_rtcp/source/rtcp_sender.cc +++ b/src/modules/rtp_rtcp/source/rtcp_sender.cc @@ -10,15 +10,15 @@ #include "rtcp_sender.h" -#include // assert -#include // rand -#include // memcpy +#include // memcpy +#include // assert +#include // rand +#include "trace.h" #include "common_types.h" -#include "modules/remote_bitrate_estimator/remote_rate_control.h" -#include "modules/rtp_rtcp/source/rtp_rtcp_impl.h" -#include "system_wrappers/interface/critical_section_wrapper.h" -#include "system_wrappers/interface/trace.h" +#include "critical_section_wrapper.h" + +#include "rtp_rtcp_impl.h" namespace webrtc { @@ -66,10 +66,12 @@ RTCPSender::RTCPSender(const WebRtc_Word32 id, _sizeRembSSRC(0), _rembSSRC(NULL), _rembBitrate(0), + _bitrate_observer(NULL), _tmmbrHelp(), _tmmbr_Send(0), _packetOH_Send(0), + _remoteRateControl(), _appSend(false), _appSubType(0), @@ -128,7 +130,7 @@ RTCPSender::Init() _sequenceNumberFIR = 0; _tmmbr_Send = 0; _packetOH_Send = 0; - //_remoteRateControl.Reset(); + _remoteRateControl.Reset(); _nextTimeToSendRTCP = 0; _CSRCs = 0; _appSend = false; @@ -259,6 +261,22 @@ RTCPSender::SetREMBData(const WebRtc_UWord32 bitrate, return 0; } +bool RTCPSender::SetRemoteBitrateObserver(RtpRemoteBitrateObserver* observer) { + CriticalSectionScoped lock(_criticalSectionRTCPSender); + if (observer && _bitrate_observer) { + return false; + } + _bitrate_observer = observer; + return true; +} + +void RTCPSender::UpdateRemoteBitrateEstimate(unsigned int target_bitrate) { + CriticalSectionScoped lock(_criticalSectionRTCPSender); + if (_bitrate_observer) { + _bitrate_observer->OnReceiveBitrateChanged(_remoteSSRC, target_bitrate); + } +} + bool RTCPSender::TMMBR() const { @@ -309,7 +327,7 @@ RTCPSender::SetRemoteSSRC( const WebRtc_UWord32 ssrc) { CriticalSectionScoped lock(_criticalSectionRTCPSender); _remoteSSRC = ssrc; - //_remoteRateControl.Reset(); + _remoteRateControl.Reset(); return 0; } @@ -1086,11 +1104,25 @@ RTCPSender::BuildREMB(WebRtc_UWord8* rtcpbuffer, WebRtc_UWord32& pos) return 0; } -void -RTCPSender::SetTargetBitrate(unsigned int target_bitrate) +WebRtc_UWord32 +RTCPSender::CalculateNewTargetBitrate(WebRtc_UWord32 RTT) { CriticalSectionScoped lock(_criticalSectionRTCPSender); + WebRtc_UWord32 target_bitrate = + _remoteRateControl.UpdateBandwidthEstimate(RTT, _clock.GetTimeInMS()); _tmmbr_Send = target_bitrate / 1000; + return target_bitrate; +} + +WebRtc_UWord32 RTCPSender::LatestBandwidthEstimate() const { + CriticalSectionScoped lock(_criticalSectionRTCPSender); + return _remoteRateControl.LatestEstimate(); +} + +bool +RTCPSender::ValidBitrateEstimate() const { + CriticalSectionScoped lock(_criticalSectionRTCPSender); + return _remoteRateControl.ValidEstimate(); } WebRtc_Word32 @@ -2131,4 +2163,12 @@ RTCPSender::SetTMMBN(const TMMBRSet* boundingSet, } return -1; } + +RateControlRegion +RTCPSender::UpdateOverUseState(const RateControlInput& rateControlInput, bool& firstOverUse) +{ + CriticalSectionScoped lock(_criticalSectionRTCPSender); + return _remoteRateControl.Update(rateControlInput, firstOverUse, + _clock.GetTimeInMS()); +} } // namespace webrtc diff --git a/src/modules/rtp_rtcp/source/rtcp_sender.h b/src/modules/rtp_rtcp/source/rtcp_sender.h index e31ca0715..e3441da34 100644 --- a/src/modules/rtp_rtcp/source/rtcp_sender.h +++ b/src/modules/rtp_rtcp/source/rtcp_sender.h @@ -17,10 +17,8 @@ #include "rtcp_utility.h" #include "rtp_utility.h" #include "rtp_rtcp_defines.h" -#include "scoped_ptr.h" +#include "remote_rate_control.h" #include "tmmbr_help.h" -#include "modules/remote_bitrate_estimator/include/bwe_defines.h" -#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" namespace webrtc { @@ -89,6 +87,10 @@ public: const WebRtc_UWord8 numberOfSSRC, const WebRtc_UWord32* SSRC); + bool SetRemoteBitrateObserver(RtpRemoteBitrateObserver* observer); + + void UpdateRemoteBitrateEstimate(unsigned int target_bitrate); + /* * TMMBR */ @@ -122,7 +124,19 @@ public: WebRtc_Word32 SetCSRCStatus(const bool include); - void SetTargetBitrate(unsigned int target_bitrate); + /* + * New bandwidth estimation + */ + + RateControlRegion UpdateOverUseState(const RateControlInput& rateControlInput, bool& firstOverUse); + + WebRtc_UWord32 CalculateNewTargetBitrate(WebRtc_UWord32 RTT); + + WebRtc_UWord32 LatestBandwidthEstimate() const; + + // Returns true if there is a valid estimate of the incoming bitrate, false + // otherwise. + bool ValidBitrateEstimate() const; private: WebRtc_Word32 SendToNetwork(const WebRtc_UWord8* dataBuffer, @@ -226,10 +240,12 @@ private: WebRtc_UWord8 _sizeRembSSRC; WebRtc_UWord32* _rembSSRC; WebRtc_UWord32 _rembBitrate; + RtpRemoteBitrateObserver* _bitrate_observer; TMMBRHelp _tmmbrHelp; WebRtc_UWord32 _tmmbr_Send; WebRtc_UWord32 _packetOH_Send; + RemoteRateControl _remoteRateControl; // APP bool _appSend; diff --git a/src/modules/rtp_rtcp/source/rtcp_sender_unittest.cc b/src/modules/rtp_rtcp/source/rtcp_sender_unittest.cc index 0b14547b7..17fbafc72 100644 --- a/src/modules/rtp_rtcp/source/rtcp_sender_unittest.cc +++ b/src/modules/rtp_rtcp/source/rtcp_sender_unittest.cc @@ -13,16 +13,13 @@ * This file includes unit tests for the RTCPSender. */ -#include #include #include "common_types.h" -#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" -#include "modules/remote_bitrate_estimator/include/mock/mock_remote_bitrate_observer.h" -#include "modules/rtp_rtcp/source/rtcp_receiver.h" -#include "modules/rtp_rtcp/source/rtcp_sender.h" -#include "modules/rtp_rtcp/source/rtp_utility.h" -#include "modules/rtp_rtcp/source/rtp_rtcp_impl.h" +#include "rtp_utility.h" +#include "rtcp_sender.h" +#include "rtcp_receiver.h" +#include "rtp_rtcp_impl.h" namespace webrtc { @@ -97,9 +94,7 @@ class TestTransport : public Transport, class RtcpSenderTest : public ::testing::Test { protected: - RtcpSenderTest() - : remote_bitrate_observer_(), - remote_bitrate_estimator_(&remote_bitrate_observer_) { + RtcpSenderTest() { system_clock_ = ModuleRTPUtility::GetSystemClock(); test_transport_ = new TestTransport(); @@ -109,7 +104,6 @@ class RtcpSenderTest : public ::testing::Test { configuration.clock = system_clock_; configuration.incoming_data = test_transport_; configuration.outgoing_transport = test_transport_; - configuration.remote_bitrate_estimator = &remote_bitrate_estimator_; rtp_rtcp_impl_ = new ModuleRtpRtcpImpl(configuration); rtcp_sender_ = new RTCPSender(0, false, system_clock_, rtp_rtcp_impl_); @@ -138,8 +132,6 @@ class RtcpSenderTest : public ::testing::Test { RTCPSender* rtcp_sender_; RTCPReceiver* rtcp_receiver_; TestTransport* test_transport_; - MockRemoteBitrateObserver remote_bitrate_observer_; - RemoteBitrateEstimator remote_bitrate_estimator_; enum {kMaxPacketLength = 1500}; uint8_t packet_[kMaxPacketLength]; diff --git a/src/modules/rtp_rtcp/source/rtp_receiver.cc b/src/modules/rtp_rtcp/source/rtp_receiver.cc index 6d927a55d..7c5439eea 100644 --- a/src/modules/rtp_rtcp/source/rtp_receiver.cc +++ b/src/modules/rtp_rtcp/source/rtp_receiver.cc @@ -32,10 +32,9 @@ using ModuleRTPUtility::VideoPayload; RTPReceiver::RTPReceiver(const WebRtc_Word32 id, const bool audio, RtpRtcpClock* clock, - RemoteBitrateEstimator* remote_bitrate, ModuleRtpRtcpImpl* owner) : RTPReceiverAudio(id), - RTPReceiverVideo(id, remote_bitrate, owner), + RTPReceiverVideo(id, owner), Bitrate(clock), _id(id), _audio(audio), @@ -1084,6 +1083,7 @@ void RTPReceiver::CheckSSRCChanged(const WebRtcRTPHeader* rtpHeader) { // reset last report ResetStatistics(); + RTPReceiverVideo::ResetOverUseDetector(); _lastReceivedTimestamp = 0; _lastReceivedSequenceNumber = 0; diff --git a/src/modules/rtp_rtcp/source/rtp_receiver.h b/src/modules/rtp_rtcp/source/rtp_receiver.h index e8bd7f364..d6ecfaaaf 100644 --- a/src/modules/rtp_rtcp/source/rtp_receiver.h +++ b/src/modules/rtp_rtcp/source/rtp_receiver.h @@ -35,7 +35,6 @@ public: RTPReceiver(const WebRtc_Word32 id, const bool audio, RtpRtcpClock* clock, - RemoteBitrateEstimator* remote_bitrate, ModuleRtpRtcpImpl* owner); virtual ~RTPReceiver(); diff --git a/src/modules/rtp_rtcp/source/rtp_receiver_video.cc b/src/modules/rtp_rtcp/source/rtp_receiver_video.cc index 807a52cfc..94d49e058 100644 --- a/src/modules/rtp_rtcp/source/rtp_receiver_video.cc +++ b/src/modules/rtp_rtcp/source/rtp_receiver_video.cc @@ -26,8 +26,20 @@ WebRtc_UWord32 BitRateBPS(WebRtc_UWord16 x ) return (x & 0x3fff) * WebRtc_UWord32(pow(10.0f,(2 + (x >> 14)))); } +RTPReceiverVideo::RTPReceiverVideo() + : _id(0), + _rtpRtcp(NULL), + _criticalSectionReceiverVideo( + CriticalSectionWrapper::CreateCriticalSection()), + _currentFecFrameDecoded(false), + _receiveFEC(NULL), + _overUseDetector(), + _videoBitRate(), + _lastBitRateChange(0), + _packetOverHead(28) { +} + RTPReceiverVideo::RTPReceiverVideo(const WebRtc_Word32 id, - RemoteBitrateEstimator* remote_bitrate, ModuleRtpRtcpImpl* owner) : _id(id), _rtpRtcp(owner), @@ -35,7 +47,9 @@ RTPReceiverVideo::RTPReceiverVideo(const WebRtc_Word32 id, CriticalSectionWrapper::CreateCriticalSection()), _currentFecFrameDecoded(false), _receiveFEC(NULL), - remote_bitrate_(remote_bitrate), + _overUseDetector(), + _videoBitRate(), + _lastBitRateChange(0), _packetOverHead(28) { } @@ -73,6 +87,12 @@ ModuleRTPUtility::Payload* RTPReceiverVideo::RegisterReceiveVideoPayload( return payload; } +void RTPReceiverVideo::ResetOverUseDetector() { + _overUseDetector.Reset(); + _videoBitRate.Init(); + _lastBitRateChange = 0; +} + // we have no critext when calling this // we are not allowed to have any critsects when calling // CallbackOfReceivedPayloadData @@ -89,15 +109,14 @@ WebRtc_Word32 RTPReceiverVideo::ParseVideoCodecSpecific( _criticalSectionReceiverVideo->Enter(); + _videoBitRate.Update(payloadDataLength + rtpHeader->header.paddingLength, + nowMS); + // Add headers, ideally we would like to include for instance // Ethernet header here as well. const WebRtc_UWord16 packetSize = payloadDataLength + _packetOverHead + rtpHeader->header.headerLength + rtpHeader->header.paddingLength; - remote_bitrate_->IncomingPacket(rtpHeader->header.ssrc, - packetSize, - nowMS, - rtpHeader->header.timestamp, - -1); + _overUseDetector.Update(*rtpHeader, packetSize, nowMS); if (isRED) { if(_receiveFEC == NULL) { @@ -135,6 +154,24 @@ WebRtc_Word32 RTPReceiverVideo::ParseVideoCodecSpecific( payloadDataLength, videoType); } + + // Update the remote rate control object and update the overuse + // detector with the current rate control region. + _criticalSectionReceiverVideo->Enter(); + const RateControlInput input(_overUseDetector.State(), + _videoBitRate.BitRate(nowMS), + _overUseDetector.NoiseVar()); + _criticalSectionReceiverVideo->Leave(); + + // Call the callback outside critical section + if (_rtpRtcp) { + const RateControlRegion region = _rtpRtcp->OnOverUseStateUpdate(input); + + _criticalSectionReceiverVideo->Enter(); + _overUseDetector.SetRateControlRegion(region); + _criticalSectionReceiverVideo->Leave(); + } + return retVal; } diff --git a/src/modules/rtp_rtcp/source/rtp_receiver_video.h b/src/modules/rtp_rtcp/source/rtp_receiver_video.h index 140563074..ce3229374 100644 --- a/src/modules/rtp_rtcp/source/rtp_receiver_video.h +++ b/src/modules/rtp_rtcp/source/rtp_receiver_video.h @@ -16,11 +16,9 @@ #include "typedefs.h" -#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" -#include "modules/remote_bitrate_estimator/overuse_detector.h" -#include "modules/remote_bitrate_estimator/remote_rate_control.h" +#include "overuse_detector.h" +#include "remote_rate_control.h" #include "Bitrate.h" -#include "scoped_ptr.h" namespace webrtc { class ReceiverFEC; @@ -29,9 +27,8 @@ class CriticalSectionWrapper; class RTPReceiverVideo { public: - RTPReceiverVideo(const WebRtc_Word32 id, - RemoteBitrateEstimator* remote_bitrate, - ModuleRtpRtcpImpl* owner); + RTPReceiverVideo(); + RTPReceiverVideo(const WebRtc_Word32 id, ModuleRtpRtcpImpl* owner); virtual ~RTPReceiverVideo(); @@ -58,6 +55,8 @@ class RTPReceiverVideo { void SetPacketOverHead(WebRtc_UWord16 packetOverHead); protected: + void ResetOverUseDetector(); + virtual WebRtc_Word32 CallbackOfReceivedPayloadData( const WebRtc_UWord8* payloadData, const WebRtc_UWord16 payloadSize, @@ -107,7 +106,9 @@ class RTPReceiverVideo { ReceiverFEC* _receiveFEC; // BWE - RemoteBitrateEstimator* remote_bitrate_; + OverUseDetector _overUseDetector; + BitRateStats _videoBitRate; + WebRtc_Word64 _lastBitRateChange; WebRtc_UWord16 _packetOverHead; }; } // namespace webrtc diff --git a/src/modules/rtp_rtcp/source/rtp_rtcp.gypi b/src/modules/rtp_rtcp/source/rtp_rtcp.gypi index 5c47bd39a..2017dff9c 100644 --- a/src/modules/rtp_rtcp/source/rtp_rtcp.gypi +++ b/src/modules/rtp_rtcp/source/rtp_rtcp.gypi @@ -13,7 +13,6 @@ 'type': '<(library)', 'dependencies': [ '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers', - '<(webrtc_root)/modules/modules.gyp:remote_bitrate_estimator', ], 'include_dirs': [ '../interface', @@ -68,8 +67,12 @@ 'forward_error_correction.h', 'forward_error_correction_internal.cc', 'forward_error_correction_internal.h', + 'overuse_detector.cc', + 'overuse_detector.h', 'producer_fec.cc', 'producer_fec.h', + 'remote_rate_control.cc', + 'remote_rate_control.h', 'rtp_packet_history.cc', 'rtp_packet_history.h', 'rtp_receiver_video.cc', diff --git a/src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc index d86ccee43..4a2a21742 100644 --- a/src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc +++ b/src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc @@ -54,7 +54,7 @@ RtpRtcp* RtpRtcp::CreateRtpRtcp(const RtpRtcp::Configuration& configuration) { ModuleRtpRtcpImpl::ModuleRtpRtcpImpl(const Configuration& configuration) : _rtpSender(configuration.id, configuration.audio, configuration.clock), _rtpReceiver(configuration.id, configuration.audio, configuration.clock, - configuration.remote_bitrate_estimator, this), + this), _rtcpSender(configuration.id, configuration.audio, configuration.clock, this), _rtcpReceiver(configuration.id, configuration.clock, this), @@ -80,8 +80,7 @@ ModuleRtpRtcpImpl::ModuleRtpRtcpImpl(const Configuration& configuration) _nackLastTimeSent(0), _nackLastSeqNumberSent(0), _simulcast(false), - _keyFrameReqMethod(kKeyFrameReqFirRtp), - remote_bitrate_(configuration.remote_bitrate_estimator) + _keyFrameReqMethod(kKeyFrameReqFirRtp) #ifdef MATLAB , _plot1(NULL) #endif @@ -103,6 +102,8 @@ ModuleRtpRtcpImpl::ModuleRtpRtcpImpl(const Configuration& configuration) _rtpSender.RegisterSendTransport(configuration.outgoing_transport); _rtcpSender.RegisterSendTransport(configuration.outgoing_transport); + _rtcpSender.SetRemoteBitrateObserver(configuration.bitrate_observer); + // make sure that RTCP objects are aware of our SSRC WebRtc_UWord32 SSRC = _rtpSender.SSRC(); _rtcpSender.SetSSRC(SSRC); @@ -223,16 +224,15 @@ WebRtc_Word32 ModuleRtpRtcpImpl::Process() { // default module or no RTCP received yet. max_rtt = kDefaultRtt; } - remote_bitrate_->SetRtt(max_rtt); - remote_bitrate_->UpdateEstimate(_rtpReceiver.SSRC(), now); - if (TMMBR()) { - unsigned int target_bitrate = 0; - if (remote_bitrate_->LatestEstimate(_rtpReceiver.SSRC(), - &target_bitrate)) { - _rtcpSender.SetTargetBitrate(target_bitrate); + if (_rtcpSender.ValidBitrateEstimate()) { + if (REMB()) { + uint32_t target_bitrate = + _rtcpSender.CalculateNewTargetBitrate(max_rtt); + _rtcpSender.UpdateRemoteBitrateEstimate(target_bitrate); + } else if (TMMBR()) { + _rtcpSender.CalculateNewTargetBitrate(max_rtt); } } - _rtcpSender.SendRTCP(kRtcpReport); } @@ -1882,12 +1882,35 @@ void ModuleRtpRtcpImpl::BitrateSent(WebRtc_UWord32* totalRate, int ModuleRtpRtcpImpl::EstimatedReceiveBandwidth( WebRtc_UWord32* available_bandwidth) const { - if (!remote_bitrate_->LatestEstimate(_rtpReceiver.SSRC(), - available_bandwidth)) + if (!_rtcpSender.ValidBitrateEstimate()) return -1; + *available_bandwidth = _rtcpSender.LatestBandwidthEstimate(); return 0; } +RateControlRegion ModuleRtpRtcpImpl::OnOverUseStateUpdate( + const RateControlInput& rateControlInput) { + + bool firstOverUse = false; + RateControlRegion region = _rtcpSender.UpdateOverUseState(rateControlInput, + firstOverUse); + if (firstOverUse) { + // Send TMMBR or REMB immediately. + WebRtc_UWord16 RTT = 0; + _rtcpReceiver.RTT(_rtpReceiver.SSRC(), &RTT, NULL, NULL, NULL); + // About to send TMMBR, first run remote rate control + // to get a target bit rate. + unsigned int target_bitrate = + _rtcpSender.CalculateNewTargetBitrate(RTT); + if (REMB()) { + _rtcpSender.UpdateRemoteBitrateEstimate(target_bitrate); + } else if (TMMBR()) { + _rtcpSender.SendRTCP(kRtcpTmmbr); + } + } + return region; +} + // bad state of RTP receiver request a keyframe void ModuleRtpRtcpImpl::OnRequestIntraFrame() { RequestKeyFrame(); diff --git a/src/modules/rtp_rtcp/source/rtp_rtcp_impl.h b/src/modules/rtp_rtcp/source/rtp_rtcp_impl.h index fbabb1b98..8724d78b3 100644 --- a/src/modules/rtp_rtcp/source/rtp_rtcp_impl.h +++ b/src/modules/rtp_rtcp/source/rtp_rtcp_impl.h @@ -428,6 +428,8 @@ class ModuleRtpRtcpImpl : public RtpRtcp { virtual WebRtc_UWord32 SendTimeOfSendReport(const WebRtc_UWord32 sendReport); + virtual RateControlRegion OnOverUseStateUpdate(const RateControlInput& rateControlInput); + // good state of RTP receiver inform sender virtual WebRtc_Word32 SendRTCPReferencePictureSelection(const WebRtc_UWord64 pictureID); @@ -504,8 +506,6 @@ private: VideoCodec _sendVideoCodec; KeyFrameRequestMethod _keyFrameReqMethod; - RemoteBitrateEstimator* remote_bitrate_; - #ifdef MATLAB MatlabPlot* _plot1; #endif diff --git a/src/video_engine/vie_channel.cc b/src/video_engine/vie_channel.cc index 3bd3b273a..0e0f69034 100644 --- a/src/video_engine/vie_channel.cc +++ b/src/video_engine/vie_channel.cc @@ -38,7 +38,7 @@ ViEChannel::ViEChannel(WebRtc_Word32 channel_id, ProcessThread& module_process_thread, RtcpIntraFrameObserver* intra_frame_observer, RtcpBandwidthObserver* bandwidth_observer, - RemoteBitrateEstimator* remote_bitrate_estimator, + RtpRemoteBitrateObserver* bitrate_observer, RtpRtcp* default_rtp_rtcp) : ViEFrameProviderBase(channel_id, engine_id), channel_id_(channel_id), @@ -91,7 +91,7 @@ ViEChannel::ViEChannel(WebRtc_Word32 channel_id, configuration.rtcp_feedback = this; configuration.intra_frame_callback = intra_frame_observer; configuration.bandwidth_callback = bandwidth_observer; - configuration.remote_bitrate_estimator = remote_bitrate_estimator; + configuration.bitrate_observer = bitrate_observer; rtp_rtcp_.reset(RtpRtcp::CreateRtpRtcp(configuration)); vie_receiver_.SetRtpRtcpModule(rtp_rtcp_.get()); diff --git a/src/video_engine/vie_channel.h b/src/video_engine/vie_channel.h index 92bf0be71..b450d244c 100644 --- a/src/video_engine/vie_channel.h +++ b/src/video_engine/vie_channel.h @@ -15,7 +15,6 @@ #include -#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" #include "modules/rtp_rtcp/interface/rtp_rtcp_defines.h" #include "modules/udp_transport/interface/udp_transport.h" #include "modules/video_coding/main/interface/video_coding_defines.h" @@ -64,7 +63,7 @@ class ViEChannel ProcessThread& module_process_thread, RtcpIntraFrameObserver* intra_frame_observer, RtcpBandwidthObserver* bandwidth_observer, - RemoteBitrateEstimator* remote_bitrate_estimator, + RtpRemoteBitrateObserver* bitrate_observer, RtpRtcp* default_rtp_rtcp); ~ViEChannel(); diff --git a/src/video_engine/vie_channel_group.cc b/src/video_engine/vie_channel_group.cc index e26e19c1c..2555286b8 100644 --- a/src/video_engine/vie_channel_group.cc +++ b/src/video_engine/vie_channel_group.cc @@ -11,7 +11,6 @@ #include "video_engine/vie_channel_group.h" #include "modules/bitrate_controller/include/bitrate_controller.h" -#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" #include "modules/rtp_rtcp/interface/rtp_rtcp.h" #include "video_engine/vie_channel.h" #include "video_engine/vie_encoder.h" @@ -21,8 +20,7 @@ namespace webrtc { ChannelGroup::ChannelGroup(ProcessThread* process_thread) : remb_(new VieRemb(process_thread)), - bitrate_controller_(BitrateController::CreateBitrateController()), - remote_bitrate_estimator_(new RemoteBitrateEstimator(remb_.get())) { + bitrate_controller_(BitrateController::CreateBitrateController()) { } ChannelGroup::~ChannelGroup() { @@ -33,9 +31,8 @@ void ChannelGroup::AddChannel(int channel_id) { channels_.insert(channel_id); } -void ChannelGroup::RemoveChannel(int channel_id, unsigned int ssrc) { +void ChannelGroup::RemoveChannel(int channel_id) { channels_.erase(channel_id); - remote_bitrate_estimator_->RemoveStream(ssrc); } bool ChannelGroup::HasChannel(int channel_id) { @@ -46,12 +43,12 @@ bool ChannelGroup::Empty() { return channels_.empty(); } -BitrateController* ChannelGroup::GetBitrateController() { - return bitrate_controller_.get(); +RtpRemoteBitrateObserver* ChannelGroup::GetRtpRemoteBitrateObserver() { + return remb_.get(); } -RemoteBitrateEstimator* ChannelGroup::GetRemoteBitrateEstimator() { - return remote_bitrate_estimator_.get(); +BitrateController* ChannelGroup::GetBitrateController() { + return bitrate_controller_.get(); } bool ChannelGroup::SetChannelRembStatus(int channel_id, @@ -67,7 +64,7 @@ bool ChannelGroup::SetChannelRembStatus(int channel_id, } else if (channel) { channel->EnableRemb(false); } - // Update the REMB instance with necessary RTP modules. + // Update the remb instance with necesary RTp modules. RtpRtcp* rtp_module = channel->rtp_rtcp(); if (sender) { remb_->AddRembSender(rtp_module); diff --git a/src/video_engine/vie_channel_group.h b/src/video_engine/vie_channel_group.h index 0fb777cd2..279a5560e 100644 --- a/src/video_engine/vie_channel_group.h +++ b/src/video_engine/vie_channel_group.h @@ -19,8 +19,7 @@ namespace webrtc { class BitrateController; class ProcessThread; -class RemoteBitrateEstimator; -class RemoteBitrateObserver; +class RtpRemoteBitrateObserver; class ViEChannel; class ViEEncoder; class VieRemb; @@ -33,7 +32,7 @@ class ChannelGroup { ~ChannelGroup(); void AddChannel(int channel_id); - void RemoveChannel(int channel_id, unsigned int ssrc); + void RemoveChannel(int channel_id); bool HasChannel(int channel_id); bool Empty(); @@ -44,14 +43,14 @@ class ChannelGroup { ViEEncoder* encoder); BitrateController* GetBitrateController(); - RemoteBitrateEstimator* GetRemoteBitrateEstimator(); + + RtpRemoteBitrateObserver* GetRtpRemoteBitrateObserver(); private: typedef std::set ChannelSet; scoped_ptr remb_; scoped_ptr bitrate_controller_; - scoped_ptr remote_bitrate_estimator_; ChannelSet channels_; }; diff --git a/src/video_engine/vie_channel_manager.cc b/src/video_engine/vie_channel_manager.cc index c51f527df..777bcccc2 100644 --- a/src/video_engine/vie_channel_manager.cc +++ b/src/video_engine/vie_channel_manager.cc @@ -97,12 +97,13 @@ int ViEChannelManager::CreateChannel(int& channel_id) { RtcpBandwidthObserver* bandwidth_observer = bitrate_controller->CreateRtcpBandwidthObserver(); - RemoteBitrateEstimator* remote_bitrate_estimator = - group->GetRemoteBitrateEstimator(); + + RtpRemoteBitrateObserver* bitrate_observer = + group->GetRtpRemoteBitrateObserver(); if (!(vie_encoder->Init() && CreateChannelObject(new_channel_id, vie_encoder, bandwidth_observer, - remote_bitrate_estimator))) { + bitrate_observer))) { delete vie_encoder; vie_encoder = NULL; ReturnChannelId(new_channel_id); @@ -135,8 +136,9 @@ int ViEChannelManager::CreateChannel(int& channel_id, RtcpBandwidthObserver* bandwidth_observer = bitrate_controller->CreateRtcpBandwidthObserver(); - RemoteBitrateEstimator* remote_bitrate_estimator = - channel_group->GetRemoteBitrateEstimator(); + + RtpRemoteBitrateObserver* bitrate_observer = + channel_group->GetRtpRemoteBitrateObserver(); ViEEncoder* vie_encoder = NULL; if (sender) { @@ -146,8 +148,7 @@ int ViEChannelManager::CreateChannel(int& channel_id, bitrate_controller); if (!(vie_encoder->Init() && CreateChannelObject(new_channel_id, vie_encoder, - bandwidth_observer, - remote_bitrate_estimator))) { + bandwidth_observer, bitrate_observer))) { delete vie_encoder; vie_encoder = NULL; } @@ -155,7 +156,7 @@ int ViEChannelManager::CreateChannel(int& channel_id, vie_encoder = ViEEncoderPtr(original_channel); assert(vie_encoder); if (!CreateChannelObject(new_channel_id, vie_encoder, bandwidth_observer, - remote_bitrate_estimator)) { + bitrate_observer)) { vie_encoder = NULL; } } @@ -201,9 +202,7 @@ int ViEChannelManager::DeleteChannel(int channel_id) { group = FindGroup(channel_id); group->SetChannelRembStatus(channel_id, false, false, vie_channel, vie_encoder); - unsigned int ssrc = 0; - vie_channel->GetRemoteSSRC(ssrc); - group->RemoveChannel(channel_id, ssrc); + group->RemoveChannel(channel_id); // Check if other channels are using the same encoder. if (ChannelUsingViEEncoder(channel_id)) { @@ -327,7 +326,7 @@ bool ViEChannelManager::CreateChannelObject( int channel_id, ViEEncoder* vie_encoder, RtcpBandwidthObserver* bandwidth_observer, - RemoteBitrateEstimator* remote_bitrate_estimator) { + RtpRemoteBitrateObserver* bitrate_observer) { // Register the channel at the encoder. RtpRtcp* send_rtp_rtcp_module = vie_encoder->SendRtpRtcpModule(); @@ -336,7 +335,7 @@ bool ViEChannelManager::CreateChannelObject( *module_process_thread_, vie_encoder, bandwidth_observer, - remote_bitrate_estimator, + bitrate_observer, send_rtp_rtcp_module); if (vie_channel->Init() != 0) { WEBRTC_TRACE(kTraceError, kTraceVideo, ViEId(engine_id_), diff --git a/src/video_engine/vie_channel_manager.h b/src/video_engine/vie_channel_manager.h index e61e74131..a0c9382a5 100644 --- a/src/video_engine/vie_channel_manager.h +++ b/src/video_engine/vie_channel_manager.h @@ -78,7 +78,7 @@ class ViEChannelManager: private ViEManagerBase { // protected. bool CreateChannelObject(int channel_id, ViEEncoder* vie_encoder, RtcpBandwidthObserver* bandwidth_observer, - RemoteBitrateEstimator* remote_bitrate_estimator); + RtpRemoteBitrateObserver* bitrate_observer); // Used by ViEChannelScoped, forcing a manager user to use scoped. // Returns a pointer to the channel with id 'channelId'. diff --git a/src/video_engine/vie_remb.h b/src/video_engine/vie_remb.h index 45921065a..dfcdf2df1 100644 --- a/src/video_engine/vie_remb.h +++ b/src/video_engine/vie_remb.h @@ -22,7 +22,6 @@ #include #include "modules/interface/module.h" -#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" #include "modules/rtp_rtcp/interface/rtp_rtcp_defines.h" #include "system_wrappers/interface/scoped_ptr.h" @@ -32,7 +31,7 @@ class CriticalSectionWrapper; class ProcessThread; class RtpRtcp; -class VieRemb : public RemoteBitrateObserver, public Module { +class VieRemb : public RtpRemoteBitrateObserver, public Module { public: VieRemb(ProcessThread* process_thread); ~VieRemb();