diff --git a/src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc index 957d4d06d..a18f69063 100644 --- a/src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc +++ b/src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc @@ -2363,6 +2363,15 @@ void ModuleRtpRtcpImpl::BitrateSent(WebRtc_UWord32* totalRate, // for default we need to update the send bitrate CriticalSectionScoped lock(_criticalSectionModulePtrsFeedback); + if (totalRate != NULL) + *totalRate = 0; + if (videoRate != NULL) + *videoRate = 0; + if (fecRate != NULL) + *fecRate = 0; + if (nackRate != NULL) + *nackRate = 0; + std::list::const_iterator it = _childModules.begin(); while (it != _childModules.end()) { @@ -2378,6 +2387,8 @@ void ModuleRtpRtcpImpl::BitrateSent(WebRtc_UWord32* totalRate, &childNackRate); if (totalRate != NULL && childTotalRate > *totalRate) *totalRate = childTotalRate; + if (videoRate != NULL && childVideoRate > *videoRate) + *videoRate = childVideoRate; if (fecRate != NULL && childFecRate > *fecRate) *fecRate = childFecRate; if (nackRate != NULL && childNackRate > *nackRate) diff --git a/src/modules/video_coding/main/interface/video_coding_defines.h b/src/modules/video_coding/main/interface/video_coding_defines.h index 32427702f..21973cbe1 100644 --- a/src/modules/video_coding/main/interface/video_coding_defines.h +++ b/src/modules/video_coding/main/interface/video_coding_defines.h @@ -147,11 +147,15 @@ protected: class VCMProtectionCallback { public: - virtual WebRtc_Word32 ProtectionRequest(const WebRtc_UWord8 deltaFECRate, - const WebRtc_UWord8 keyFECRate, - const bool deltaUseUepProtection, - const bool keyUseUepProtection, - const bool nack) = 0; + virtual int ProtectionRequest( + uint8_t delta_fec_rate, + uint8_t key_fec_rate, + bool delta_use_uep_protection, + bool key_use_uep_protection, + bool nack_enabled, + uint32_t* sent_video_rate_bps, + uint32_t* sent_nack_rate_bps, + uint32_t* sent_fec_rate_bps) = 0; protected: virtual ~VCMProtectionCallback() {} diff --git a/src/modules/video_coding/main/source/media_optimization.cc b/src/modules/video_coding/main/source/media_optimization.cc index 09a3a1801..b6f172ca9 100644 --- a/src/modules/video_coding/main/source/media_optimization.cc +++ b/src/modules/video_coding/main/source/media_optimization.cc @@ -22,7 +22,6 @@ _sendCodecType(kVideoCodecUnknown), _codecWidth(0), _codecHeight(0), _userFrameRate(0), -_lossProtOverhead(0), _packetLossEnc(0), _fractionLost(0), _sendStatisticsZeroEncode(0), @@ -71,7 +70,6 @@ VCMMediaOptimization::Reset() _lossProtLogic->Reset(); _sendStatisticsZeroEncode = 0; _targetBitRate = 0; - _lossProtOverhead = 0; _codecWidth = 0; _codecHeight = 0; _userFrameRate = 0; @@ -124,7 +122,7 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, _lossProtLogic->UpdateFilteredLossPr(packetLossEnc); // Rate cost of the protection methods - _lossProtOverhead = 0; + uint32_t protection_overhead_kbps = 0; // Update protection settings, when applicable if (selectedMethod) @@ -137,24 +135,40 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, // the protection method is set by the user via SetVideoProtection. _lossProtLogic->UpdateMethod(); - // Update protection callback with protection settings - UpdateProtectionCallback(selectedMethod); - - // Get the bit cost of protection method - _lossProtOverhead = static_cast - (_lossProtLogic->RequiredBitRate() + 0.5f); + // Update protection callback with protection settings. + uint32_t sent_video_rate_bps = 0; + uint32_t sent_nack_rate_bps = 0; + uint32_t sent_fec_rate_bps = 0; + // Get the bit cost of protection method, based on the amount of + // overhead data actually transmitted (including headers) the last + // second. + UpdateProtectionCallback(selectedMethod, + &sent_video_rate_bps, + &sent_nack_rate_bps, + &sent_fec_rate_bps); + uint32_t sent_total_rate_bps = sent_video_rate_bps + + sent_nack_rate_bps + sent_fec_rate_bps; + // Estimate the overhead costs of the next second as staying the same + // wrt the source bitrate. + if (sent_total_rate_bps > 0) { + protection_overhead_kbps = static_cast(bitRate * + static_cast(sent_nack_rate_bps + sent_fec_rate_bps) / + sent_total_rate_bps + 0.5); + } + // Cap the overhead estimate to 50%. + if (protection_overhead_kbps > bitRate / 2) + protection_overhead_kbps = bitRate / 2; // Get the effective packet loss for encoder ER // when applicable, should be passed to encoder via fractionLost packetLossEnc = selectedMethod->RequiredPacketLossER(); } - // Update encoding rates following protection settings - _frameDropper->SetRates(static_cast(bitRate - - _lossProtOverhead), 0); - // Source coding rate: total rate - protection overhead - _targetBitRate = bitRate - _lossProtOverhead; + _targetBitRate = bitRate - protection_overhead_kbps; + + // Update encoding rates following protection settings + _frameDropper->SetRates(static_cast(_targetBitRate), 0); if (_enableQm) { @@ -174,9 +188,11 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, return _targetBitRate; } -WebRtc_UWord32 -VCMMediaOptimization::UpdateProtectionCallback(VCMProtectionMethod - *selectedMethod) +int VCMMediaOptimization::UpdateProtectionCallback( + VCMProtectionMethod *selected_method, + uint32_t* video_rate_bps, + uint32_t* nack_overhead_rate_bps, + uint32_t* fec_overhead_rate_bps) { if (!_videoProtectionCallback) { @@ -184,29 +200,32 @@ VCMMediaOptimization::UpdateProtectionCallback(VCMProtectionMethod } // Get the FEC code rate for Key frames (set to 0 when NA) const WebRtc_UWord8 - codeRateKeyRTP = selectedMethod->RequiredProtectionFactorK(); + codeRateKeyRTP = selected_method->RequiredProtectionFactorK(); // Get the FEC code rate for Delta frames (set to 0 when NA) const WebRtc_UWord8 - codeRateDeltaRTP = selectedMethod->RequiredProtectionFactorD(); + codeRateDeltaRTP = selected_method->RequiredProtectionFactorD(); // Get the FEC-UEP protection status for Key frames: UEP on/off const bool - useUepProtectionKeyRTP = selectedMethod->RequiredUepProtectionK(); + useUepProtectionKeyRTP = selected_method->RequiredUepProtectionK(); // Get the FEC-UEP protection status for Delta frames: UEP on/off const bool - useUepProtectionDeltaRTP = selectedMethod->RequiredUepProtectionD(); + useUepProtectionDeltaRTP = selected_method->RequiredUepProtectionD(); // NACK is on for NACK and NackFec protection method: off for FEC method - bool nackStatus = (selectedMethod->Type() == kNackFec || - selectedMethod->Type() == kNack); + bool nackStatus = (selected_method->Type() == kNackFec || + selected_method->Type() == kNack); - return _videoProtectionCallback->ProtectionRequest(codeRateDeltaRTP, - codeRateKeyRTP, - useUepProtectionDeltaRTP, - useUepProtectionKeyRTP, - nackStatus); + return _videoProtectionCallback->ProtectionRequest(codeRateDeltaRTP, + codeRateKeyRTP, + useUepProtectionDeltaRTP, + useUepProtectionKeyRTP, + nackStatus, + video_rate_bps, + nack_overhead_rate_bps, + fec_overhead_rate_bps); } bool diff --git a/src/modules/video_coding/main/source/media_optimization.h b/src/modules/video_coding/main/source/media_optimization.h index 844d6edd8..b02a6bb03 100644 --- a/src/modules/video_coding/main/source/media_optimization.h +++ b/src/modules/video_coding/main/source/media_optimization.h @@ -139,7 +139,10 @@ private: /* * Update protection callback with protection settings */ - WebRtc_UWord32 UpdateProtectionCallback(VCMProtectionMethod *selectedMethod); + int UpdateProtectionCallback(VCMProtectionMethod *selected_method, + uint32_t* total_video_rate_bps, + uint32_t* nack_overhead_rate_bps, + uint32_t* fec_overhead_rate_bps); void UpdateBitRateEstimate(WebRtc_Word64 encodedLength, WebRtc_Word64 nowMs); /* @@ -168,7 +171,6 @@ private: VCMFrameDropper* _frameDropper; VCMLossProtectionLogic* _lossProtLogic; - WebRtc_UWord32 _lossProtOverhead; WebRtc_UWord8 _packetLossEnc; WebRtc_UWord8 _fractionLost; diff --git a/src/modules/video_coding/main/test/test_callbacks.cc b/src/modules/video_coding/main/test/test_callbacks.cc index 0dd9be3cd..d77c22ebe 100644 --- a/src/modules/video_coding/main/test/test_callbacks.cc +++ b/src/modules/video_coding/main/test/test_callbacks.cc @@ -438,17 +438,20 @@ VideoProtectionCallback::~VideoProtectionCallback() } WebRtc_Word32 -VideoProtectionCallback::ProtectionRequest(const WebRtc_UWord8 deltaFECRate, - const WebRtc_UWord8 keyFECRate, - const bool deltaUseUepProtection, - const bool keyUseUepProtection, - const bool nack) +VideoProtectionCallback::ProtectionRequest(WebRtc_UWord8 deltaFECRate, + WebRtc_UWord8 keyFECRate, + bool deltaUseUepProtection, + bool keyUseUepProtection, + bool nack_enabled, + WebRtc_UWord32* sent_video_rate_bps, + WebRtc_UWord32* sent_nack_rate_bps, + WebRtc_UWord32* sent_fec_rate_bps) { _deltaFECRate = deltaFECRate; _keyFECRate = keyFECRate; _deltaUseUepProtection = deltaUseUepProtection; _keyUseUepProtection = keyUseUepProtection; - if (nack == true) + if (nack_enabled) { _nack = kNackRtcp; } diff --git a/src/modules/video_coding/main/test/test_callbacks.h b/src/modules/video_coding/main/test/test_callbacks.h index deb24b4bc..e192a793a 100644 --- a/src/modules/video_coding/main/test/test_callbacks.h +++ b/src/modules/video_coding/main/test/test_callbacks.h @@ -236,11 +236,14 @@ public: VideoProtectionCallback(); virtual ~VideoProtectionCallback(); void RegisterRtpModule(RtpRtcp* rtp) {_rtp = rtp;} - WebRtc_Word32 ProtectionRequest(const WebRtc_UWord8 deltaFECRate, - const WebRtc_UWord8 keyFECRate, - const bool deltaUseUepProtection, - const bool keyUseUepProtection, - const bool nack); + WebRtc_Word32 ProtectionRequest(WebRtc_UWord8 deltaFECRate, + WebRtc_UWord8 keyFECRate, + bool deltaUseUepProtection, + bool keyUseUepProtection, + bool nack_enabled, + WebRtc_UWord32* sent_video_rate_bps, + WebRtc_UWord32* sent_nack_rate_bps, + WebRtc_UWord32* sent_fec_rate_bps); enum NACKMethod NACKMethod(); WebRtc_UWord8 FECDeltaRate(); WebRtc_UWord8 FECKeyRate(); diff --git a/src/video_engine/vie_encoder.cc b/src/video_engine/vie_encoder.cc index 9b4aec5a4..5eeda510c 100644 --- a/src/video_engine/vie_encoder.cc +++ b/src/video_engine/vie_encoder.cc @@ -649,17 +649,21 @@ WebRtc_Word32 ViEEncoder::SendData( rtp_video_hdr); } -WebRtc_Word32 ViEEncoder::ProtectionRequest(const WebRtc_UWord8 delta_fecrate, - const WebRtc_UWord8 key_fecrate, - const bool delta_use_uep_protection, - const bool key_use_uep_protection, - const bool nack) { +WebRtc_Word32 ViEEncoder::ProtectionRequest( + WebRtc_UWord8 delta_fecrate, + WebRtc_UWord8 key_fecrate, + bool delta_use_uep_protection, + bool key_use_uep_protection, + bool nack_enabled, + WebRtc_UWord32* sent_video_rate_bps, + WebRtc_UWord32* sent_nack_rate_bps, + WebRtc_UWord32* sent_fec_rate_bps) { WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceVideo, ViEId(engine_id_, channel_id_), "%s, deltaFECRate: %u, key_fecrate: %u, " - "delta_use_uep_protection: %d, key_use_uep_protection: %d, " - "nack: %d", __FUNCTION__, delta_fecrate, key_fecrate, - delta_use_uep_protection, key_use_uep_protection, nack); + "delta_use_uep_protection: %d, key_use_uep_protection: %d, ", + __FUNCTION__, delta_fecrate, key_fecrate, + delta_use_uep_protection, key_use_uep_protection); if (default_rtp_rtcp_.SetFECCodeRate(key_fecrate, delta_fecrate) != 0) { WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideo, @@ -672,6 +676,10 @@ WebRtc_Word32 ViEEncoder::ProtectionRequest(const WebRtc_UWord8 delta_fecrate, ViEId(engine_id_, channel_id_), "%s: Could not update FEC-UEP protection", __FUNCTION__); } + default_rtp_rtcp_.BitrateSent(NULL, + sent_video_rate_bps, + sent_fec_rate_bps, + sent_nack_rate_bps); return 0; } diff --git a/src/video_engine/vie_encoder.h b/src/video_engine/vie_encoder.h index 87c245128..6e72e9066 100644 --- a/src/video_engine/vie_encoder.h +++ b/src/video_engine/vie_encoder.h @@ -101,11 +101,15 @@ class ViEEncoder const RTPVideoHeader* rtp_video_hdr); // Implements VideoProtectionCallback. - virtual WebRtc_Word32 ProtectionRequest(const WebRtc_UWord8 delta_fecrate, - const WebRtc_UWord8 key_fecrate, - const bool delta_use_uep_protection, - const bool key_use_uep_protection, - const bool nack); + virtual WebRtc_Word32 ProtectionRequest( + WebRtc_UWord8 delta_fecrate, + WebRtc_UWord8 key_fecrate, + bool delta_use_uep_protection, + bool key_use_uep_protection, + bool nack_enabled, + WebRtc_UWord32* sent_video_rate_bps, + WebRtc_UWord32* sent_nack_rate_bps, + WebRtc_UWord32* sent_fec_rate_bps); // Implements VideoSendStatisticsCallback. virtual WebRtc_Word32 SendStatistics(const WebRtc_UWord32 bit_rate,