From 0e7d9d862a1b5f3cdf38b70a1260caae4baa0bcc Mon Sep 17 00:00:00 2001 From: "mikhal@webrtc.org" Date: Mon, 19 Dec 2011 19:04:49 +0000 Subject: [PATCH] Adding layer info consideration when applying FEC protection. In this first version, we hard code protection zero for non-base layer frames. As a future enhancement, an array should be passed from mediaOpt to set the protection per layer. A TODO was added in MediaOpt. Review URL: http://webrtc-codereview.appspot.com/330005 git-svn-id: http://webrtc.googlecode.com/svn/trunk@1238 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../rtp_rtcp/source/rtp_sender_video.cc | 5 ++ .../vp8/main/interface/vp8_common_types.h | 29 ++++++++ .../codecs/vp8/main/source/temporal_layers.cc | 17 +++-- .../codecs/vp8/main/source/vp8.gypi | 1 + .../main/source/media_opt_util.cc | 74 +++++++++++++------ .../video_coding/main/source/media_opt_util.h | 13 +++- .../main/source/media_optimization.cc | 20 +++-- .../main/source/media_optimization.h | 4 +- .../main/source/video_coding_impl.cc | 8 +- 9 files changed, 133 insertions(+), 38 deletions(-) create mode 100644 src/modules/video_coding/codecs/vp8/main/interface/vp8_common_types.h diff --git a/src/modules/rtp_rtcp/source/rtp_sender_video.cc b/src/modules/rtp_rtcp/source/rtp_sender_video.cc index a8a78d0b5..5a8896312 100644 --- a/src/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/src/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -436,6 +436,11 @@ RTPSenderVideo::SendVideo(const RtpVideoCodecTypes videoType, { _fecProtectionFactor = _codeRateKey; _fecUseUepProtection = _useUepProtectionKey; + } else if (videoType == kRtpVp8Video && rtpTypeHdr->VP8.temporalIdx > 0) + { + // In current version, we only apply FEC on the base layer. + _fecProtectionFactor = 0; + _fecUseUepProtection = false; } else { _fecProtectionFactor = _codeRateDelta; diff --git a/src/modules/video_coding/codecs/vp8/main/interface/vp8_common_types.h b/src/modules/video_coding/codecs/vp8/main/interface/vp8_common_types.h new file mode 100644 index 000000000..6f347cd7b --- /dev/null +++ b/src/modules/video_coding/codecs/vp8/main/interface/vp8_common_types.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_COMMON_TYPES_H_ +#define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_COMMON_TYPES_H_ + +#include "common_types.h" + +namespace webrtc { + +// Ratio allocation between temporal streams: +// Values as required for the VP8 codec (accumulating). +static const float + kVp8LayerRateAlloction[kMaxTemporalStreams][kMaxTemporalStreams] = { + {1.0f, 0, 0, 0}, // 1 layer + {0.6f, 1.0f , 0 , 0}, // 2 layers {60%, 40%} + {0.4f, 0.6f , 1.0f, 0}, // 3 layers {40%, 20%, 40%} + {0.25f, 0.4f, 0.6f, 1.0f} // 4 layers {25%, 15%, 20%, 40%} +}; + +} // namespace webrtc +#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_COMMON_TYPES_H_ diff --git a/src/modules/video_coding/codecs/vp8/main/source/temporal_layers.cc b/src/modules/video_coding/codecs/vp8/main/source/temporal_layers.cc index 2f6b5f9b4..0b52a81cb 100644 --- a/src/modules/video_coding/codecs/vp8/main/source/temporal_layers.cc +++ b/src/modules/video_coding/codecs/vp8/main/source/temporal_layers.cc @@ -13,8 +13,9 @@ #include #include -#include "module_common_types.h" -#include "video_codec_interface.h" +#include "modules/interface/module_common_types.h" +#include "modules/video_coding/codecs/interface/video_codec_interface.h" +#include "modules/video_coding/codecs/vp8/main/interface/vp8_common_types.h" #include "vpx/vpx_encoder.h" #include "vpx/vp8cx.h" @@ -47,7 +48,7 @@ bool TemporalLayers::ConfigureBitrates(int bitrateKbit, cfg->ts_periodicity = temporal_ids_length_; // Split stream 60% 40%. // Bitrate API for VP8 is the agregated bitrate for all lower layers. - cfg->ts_target_bitrate[0] = bitrateKbit * 3 / 5; + cfg->ts_target_bitrate[0] = bitrateKbit * kVp8LayerRateAlloction[1][0]; cfg->ts_target_bitrate[1] = bitrateKbit; cfg->ts_rate_decimator[0] = 2; cfg->ts_rate_decimator[1] = 1; @@ -74,8 +75,8 @@ bool TemporalLayers::ConfigureBitrates(int bitrateKbit, cfg->ts_periodicity = temporal_ids_length_; // Split stream 40% 20% 40%. // Bitrate API for VP8 is the agregated bitrate for all lower layers. - cfg->ts_target_bitrate[0] = bitrateKbit * 2 / 5; - cfg->ts_target_bitrate[1] = bitrateKbit * 3 / 5; + cfg->ts_target_bitrate[0] = bitrateKbit * kVp8LayerRateAlloction[2][0]; + cfg->ts_target_bitrate[1] = bitrateKbit * kVp8LayerRateAlloction[2][1]; cfg->ts_target_bitrate[2] = bitrateKbit; cfg->ts_rate_decimator[0] = 4; cfg->ts_rate_decimator[1] = 2; @@ -107,9 +108,9 @@ bool TemporalLayers::ConfigureBitrates(int bitrateKbit, // Bitrate API for VP8 is the agregated bitrate for all lower layers. cfg->ts_number_layers = 4; cfg->ts_periodicity = temporal_ids_length_; - cfg->ts_target_bitrate[0] = bitrateKbit / 4; - cfg->ts_target_bitrate[1] = bitrateKbit * 2 / 5; - cfg->ts_target_bitrate[2] = bitrateKbit * 3 / 5; + cfg->ts_target_bitrate[0] = bitrateKbit * kVp8LayerRateAlloction[3][0]; + cfg->ts_target_bitrate[1] = bitrateKbit * kVp8LayerRateAlloction[3][1]; + cfg->ts_target_bitrate[2] = bitrateKbit * kVp8LayerRateAlloction[3][2]; cfg->ts_target_bitrate[3] = bitrateKbit; cfg->ts_rate_decimator[0] = 8; cfg->ts_rate_decimator[1] = 4; diff --git a/src/modules/video_coding/codecs/vp8/main/source/vp8.gypi b/src/modules/video_coding/codecs/vp8/main/source/vp8.gypi index 34a52ab75..4288d822e 100644 --- a/src/modules/video_coding/codecs/vp8/main/source/vp8.gypi +++ b/src/modules/video_coding/codecs/vp8/main/source/vp8.gypi @@ -53,6 +53,7 @@ 'reference_picture_selection.h', 'reference_picture_selection.cc', '../interface/vp8.h', + '../interface/vp8_common_types.h', '../interface/vp8_simulcast.h', 'vp8.cc', 'vp8_simulcast.cc', diff --git a/src/modules/video_coding/main/source/media_opt_util.cc b/src/modules/video_coding/main/source/media_opt_util.cc index 22d2f9c6e..2310cda7c 100644 --- a/src/modules/video_coding/main/source/media_opt_util.cc +++ b/src/modules/video_coding/main/source/media_opt_util.cc @@ -8,17 +8,21 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "video_coding_defines.h" -#include "fec_tables_xor.h" -#include "er_tables_xor.h" -#include "nack_fec_tables.h" -#include "qm_select_data.h" -#include "media_opt_util.h" +#include "modules/video_coding/main/source/media_opt_util.h" #include #include #include +#include "modules/interface/module_common_types.h" +#include "modules/video_coding/codecs/vp8/main/interface/vp8_common_types.h" +#include "modules/video_coding/main/interface/video_coding_defines.h" +#include "modules/video_coding/main/source/er_tables_xor.h" +#include "modules/video_coding/main/source/fec_tables_xor.h" +#include "modules/video_coding/main/source/nack_fec_tables.h" +#include "modules/video_coding/main/source/qm_select_data.h" + + namespace webrtc { VCMProtectionMethod::VCMProtectionMethod(): @@ -345,12 +349,8 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters) // Use a smaller exponent (< 1) to control/soften system size effect. const float resolnFac = 1.0 / powf(spatialSizeToRef, 0.3f); - const float bitRate = parameters->bitRate; - const float frameRate = parameters->frameRate; + const int bitRatePerFrame = BitsPerFrame(parameters); - // Average bits per frame (units of kbits) - const WebRtc_UWord16 bitRatePerFrame = static_cast - (bitRate / frameRate); // Average number of packets per frame (source and fec): const WebRtc_UWord8 avgTotPackets = 1 + (WebRtc_UWord8) @@ -399,11 +399,17 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters) codeRateDelta = kPacketLossMax - 1; } - float adjustFec = _qmRobustness->AdjustFecFactor(codeRateDelta, - bitRate, - frameRate, - parameters->rtt, - packetLoss); + float adjustFec = 1.0f; + // Avoid additional adjustments when layers are active. + // TODO(mikhal/marco): Update adjusmtent based on layer info. + if (parameters->numLayers == 1) + { + adjustFec = _qmRobustness->AdjustFecFactor(codeRateDelta, + parameters->bitRate, + parameters->frameRate, + parameters->rtt, + packetLoss); + } codeRateDelta = static_cast(codeRateDelta * adjustFec); @@ -481,16 +487,36 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters) } // TODO (marpan): Set the UEP protection on/off for Key and Delta frames - _useUepProtectionK = _qmRobustness->SetUepProtection(codeRateKey, bitRate, - packetLoss, 0); + _useUepProtectionK = _qmRobustness->SetUepProtection(codeRateKey, + parameters->bitRate, + packetLoss, + 0); - _useUepProtectionD = _qmRobustness->SetUepProtection(codeRateDelta, bitRate, - packetLoss, 1); + _useUepProtectionD = _qmRobustness->SetUepProtection(codeRateDelta, + parameters->bitRate, + packetLoss, + 1); // DONE WITH FEC PROTECTION SETTINGS return true; } +int VCMFecMethod::BitsPerFrame(const VCMProtectionParameters* parameters) { + // When temporal layers are available FEC will only be applied on the base + // layer. + const float bitRateRatio = + kVp8LayerRateAlloction[parameters->numLayers - 1][0]; + float frameRateRatio = powf(1 / 2, parameters->numLayers - 1); + float bitRate = parameters->bitRate * bitRateRatio; + float frameRate = parameters->frameRate * frameRateRatio; + + // TODO(mikhal): Update factor following testing. + float adjustmentFactor = 1; + + // Average bits per frame (units of kbits) + return static_cast(adjustmentFactor * bitRate / frameRate); +} + bool VCMFecMethod::EffectivePacketLoss(const VCMProtectionParameters* parameters) { @@ -570,7 +596,8 @@ _packetsPerFrameKey(0.9999f), _residualPacketLossFec(0), _boostRateKey(2), _codecWidth(0), -_codecHeight(0) +_codecHeight(0), +_numLayers(1) { Reset(); } @@ -801,6 +828,10 @@ VCMLossProtectionLogic::UpdateFrameSize(WebRtc_UWord16 width, _codecHeight = height; } +void VCMLossProtectionLogic::UpdateNumLayers(int numLayers) { + _numLayers = (numLayers == 0) ? 1 : numLayers; +} + bool VCMLossProtectionLogic::UpdateMethod() { @@ -820,6 +851,7 @@ VCMLossProtectionLogic::UpdateMethod() _currentParameters.residualPacketLossFec = _residualPacketLossFec; _currentParameters.codecWidth = _codecWidth; _currentParameters.codecHeight = _codecHeight; + _currentParameters.numLayers = _numLayers; return _selectedMethod->UpdateParameters(&_currentParameters); } diff --git a/src/modules/video_coding/main/source/media_opt_util.h b/src/modules/video_coding/main/source/media_opt_util.h index c6566fb6a..1239f9de9 100644 --- a/src/modules/video_coding/main/source/media_opt_util.h +++ b/src/modules/video_coding/main/source/media_opt_util.h @@ -46,7 +46,8 @@ struct VCMProtectionParameters VCMProtectionParameters() : rtt(0), lossPr(0.0f), bitRate(0.0f), packetsPerFrame(0.0f), packetsPerFrameKey(0.0f), frameRate(0.0f), keyFrameSize(0.0f), fecRateDelta(0), fecRateKey(0), - residualPacketLossFec(0.0f), codecWidth(0), codecHeight(0) + residualPacketLossFec(0.0f), codecWidth(0), codecHeight(0), + numLayers(1) {} int rtt; @@ -61,6 +62,7 @@ struct VCMProtectionParameters float residualPacketLossFec; WebRtc_UWord16 codecWidth; WebRtc_UWord16 codecHeight; + int numLayers; }; @@ -189,6 +191,8 @@ public: void UpdateProtectionFactorD(WebRtc_UWord8 protectionFactorD); // Update FEC with protectionFactorK void UpdateProtectionFactorK(WebRtc_UWord8 protectionFactorK); + // Compute the bits per frame. Account for temporal layers when applicable. + int BitsPerFrame(const VCMProtectionParameters* parameters); }; @@ -297,6 +301,12 @@ public: // - height : The codec frame height. void UpdateFrameSize(WebRtc_UWord16 width, WebRtc_UWord16 height); + // Update the number of active layers + // + // Input: + // - numLayers : Number of layers used. + void UpdateNumLayers(int numLayers); + // The amount of packet loss to cover for with FEC. // // Input: @@ -355,6 +365,7 @@ private: WebRtc_UWord8 _boostRateKey; WebRtc_UWord16 _codecWidth; WebRtc_UWord16 _codecHeight; + int _numLayers; }; } // namespace webrtc diff --git a/src/modules/video_coding/main/source/media_optimization.cc b/src/modules/video_coding/main/source/media_optimization.cc index b6f172ca9..008b9dbed 100644 --- a/src/modules/video_coding/main/source/media_optimization.cc +++ b/src/modules/video_coding/main/source/media_optimization.cc @@ -36,7 +36,8 @@ _avgSentBitRateBps(0.0f), _keyFrameCnt(0), _deltaFrameCnt(0), _lastQMUpdateTime(0), -_lastChangeTime(0) +_lastChangeTime(0), +_numLayers(0) { memset(_sendStatistics, 0, sizeof(_sendStatistics)); memset(_incomingFrameTimes, -1, sizeof(_incomingFrameTimes)); @@ -83,6 +84,7 @@ VCMMediaOptimization::Reset() _encodedFrameSamples[i]._timeCompleteMs = -1; } _avgSentBitRateBps = 0.0f; + _numLayers = 1; return VCM_OK; } @@ -170,7 +172,7 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, // Update encoding rates following protection settings _frameDropper->SetRates(static_cast(_targetBitRate), 0); - if (_enableQm) + if (_enableQm && _numLayers == 1) { // Update QM with rates _qmResolution->UpdateRates((float)_targetBitRate, _avgSentBitRateBps, @@ -218,6 +220,8 @@ int VCMMediaOptimization::UpdateProtectionCallback( bool nackStatus = (selected_method->Type() == kNackFec || selected_method->Type() == kNack); + // TODO(Marco): Pass FEC protection values per layer. + return _videoProtectionCallback->ProtectionRequest(codeRateDeltaRTP, codeRateKeyRTP, useUepProtectionDeltaRTP, @@ -245,9 +249,13 @@ VCMMediaOptimization::SentFrameCount(VCMFrameCount &frameCount) const } WebRtc_Word32 -VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, WebRtc_Word32 maxBitRate, - WebRtc_UWord32 frameRate, WebRtc_UWord32 bitRate, - WebRtc_UWord16 width, WebRtc_UWord16 height) +VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, + WebRtc_Word32 maxBitRate, + WebRtc_UWord32 frameRate, + WebRtc_UWord32 bitRate, + WebRtc_UWord16 width, + WebRtc_UWord16 height, + int numLayers) { // Everything codec specific should be reset here since this means the codec // has changed. If native dimension values have changed, then either user @@ -263,12 +271,14 @@ VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, WebRtc_Word3 _lossProtLogic->UpdateBitRate(static_cast(bitRate)); _lossProtLogic->UpdateFrameRate(static_cast(frameRate)); _lossProtLogic->UpdateFrameSize(width, height); + _lossProtLogic->UpdateNumLayers(numLayers); _frameDropper->Reset(); _frameDropper->SetRates(static_cast(bitRate), static_cast(frameRate)); _userFrameRate = static_cast(frameRate); _codecWidth = width; _codecHeight = height; + _numLayers = (numLayers <= 1) ? 1 : numLayers; // Can also be zero. WebRtc_Word32 ret = VCM_OK; ret = _qmResolution->Initialize((float)_targetBitRate, _userFrameRate, _codecWidth, _codecHeight); diff --git a/src/modules/video_coding/main/source/media_optimization.h b/src/modules/video_coding/main/source/media_optimization.h index b02a6bb03..bb09db70c 100644 --- a/src/modules/video_coding/main/source/media_optimization.h +++ b/src/modules/video_coding/main/source/media_optimization.h @@ -65,7 +65,8 @@ public: WebRtc_UWord32 frameRate, WebRtc_UWord32 bitRate, WebRtc_UWord16 width, - WebRtc_UWord16 height); + WebRtc_UWord16 height, + int numTemporalLayers); /** * Enable protection method */ @@ -199,6 +200,7 @@ private: WebRtc_Word64 _lastQMUpdateTime; WebRtc_Word64 _lastChangeTime; // content/user triggered + int _numLayers; }; // end of VCMMediaOptimization class definition diff --git a/src/modules/video_coding/main/source/video_coding_impl.cc b/src/modules/video_coding/main/source/video_coding_impl.cc index 036478ca6..33e18cc98 100644 --- a/src/modules/video_coding/main/source/video_coding_impl.cc +++ b/src/modules/video_coding/main/source/video_coding_impl.cc @@ -366,7 +366,7 @@ VideoCodingModuleImpl::InitializeSender() _encoder = NULL; _encodedFrameCallback.SetTransportCallback(NULL); // setting default bitRate and frameRate to 0 - _mediaOpt.SetEncodingData(kVideoCodecUnknown, 0, 0, 0, 0, 0); + _mediaOpt.SetEncodingData(kVideoCodecUnknown, 0, 0, 0, 0, 0, 0); _mediaOpt.Reset(); // Resetting frame dropper return VCM_OK; } @@ -420,12 +420,16 @@ VideoCodingModuleImpl::RegisterSendCodec(const VideoCodec* sendCodec, return VCM_CODEC_ERROR; } _sendCodecType = sendCodec->codecType; + int numLayers = (_sendCodecType != kVideoCodecVP8) ? 1 : + sendCodec->codecSpecific.VP8.numberOfTemporalLayers; + _mediaOpt.SetEncodingData(_sendCodecType, sendCodec->maxBitrate, sendCodec->maxFramerate, sendCodec->startBitrate, sendCodec->width, - sendCodec->height); + sendCodec->height, + numLayers); _mediaOpt.SetMtu(maxPayloadSize); return VCM_OK;