Adding layer info consideration when applying FEC protection. In this first version, we hard code protection zero for non-base layer frames. As a future enhancement, an array should be passed from mediaOpt to set the protection per layer. A TODO was added in MediaOpt.

Review URL: http://webrtc-codereview.appspot.com/330005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1238 4adac7df-926f-26a2-2b94-8c16560cd09d

src/modules/rtp_rtcp/source/rtp_sender_video.cc

@@ -436,6 +436,11 @@ RTPSenderVideo::SendVideo(const RtpVideoCodecTypes videoType,
     {
         _fecProtectionFactor = _codeRateKey;
         _fecUseUepProtection = _useUepProtectionKey;
+    } else if (videoType == kRtpVp8Video && rtpTypeHdr->VP8.temporalIdx > 0)
+    {
+        // In current version, we only apply FEC on the base layer.
+        _fecProtectionFactor = 0;
+        _fecUseUepProtection = false;
     } else
     {
         _fecProtectionFactor = _codeRateDelta;

src/modules/video_coding/codecs/vp8/main/interface/vp8_common_types.h

@@ -0,0 +1,29 @@
+/*
+ *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_COMMON_TYPES_H_
+#define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_COMMON_TYPES_H_
+
+#include "common_types.h"
+
+namespace webrtc {
+
+// Ratio allocation between temporal streams:
+// Values as required for the VP8 codec (accumulating).
+static const float
+  kVp8LayerRateAlloction[kMaxTemporalStreams][kMaxTemporalStreams] = {
+    {1.0f, 0, 0, 0},  // 1 layer
+    {0.6f, 1.0f , 0 , 0},  // 2 layers {60%, 40%}
+    {0.4f, 0.6f , 1.0f, 0},  // 3 layers {40%, 20%, 40%}
+    {0.25f, 0.4f, 0.6f, 1.0f}  // 4 layers {25%, 15%, 20%, 40%}
+};
+
+}  // namespace webrtc
+#endif  // WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_COMMON_TYPES_H_

src/modules/video_coding/codecs/vp8/main/source/temporal_layers.cc

@@ -13,8 +13,9 @@
 #include <string.h>
 #include <cassert>
 
-#include "module_common_types.h"
+#include "modules/interface/module_common_types.h"
-#include "video_codec_interface.h"
+#include "modules/video_coding/codecs/interface/video_codec_interface.h"
+#include "modules/video_coding/codecs/vp8/main/interface/vp8_common_types.h"
 
 #include "vpx/vpx_encoder.h"
 #include "vpx/vp8cx.h"
@@ -47,7 +48,7 @@ bool TemporalLayers::ConfigureBitrates(int bitrateKbit,
       cfg->ts_periodicity = temporal_ids_length_;
       // Split stream 60% 40%.
       // Bitrate API for VP8 is the agregated bitrate for all lower layers.
-      cfg->ts_target_bitrate[0] = bitrateKbit * 3 / 5;
+      cfg->ts_target_bitrate[0] = bitrateKbit * kVp8LayerRateAlloction[1][0];
       cfg->ts_target_bitrate[1] = bitrateKbit;
       cfg->ts_rate_decimator[0] = 2;
       cfg->ts_rate_decimator[1] = 1;
@@ -74,8 +75,8 @@ bool TemporalLayers::ConfigureBitrates(int bitrateKbit,
       cfg->ts_periodicity = temporal_ids_length_;
       // Split stream 40% 20% 40%.
       // Bitrate API for VP8 is the agregated bitrate for all lower layers.
-      cfg->ts_target_bitrate[0] = bitrateKbit * 2 / 5;
+      cfg->ts_target_bitrate[0] = bitrateKbit * kVp8LayerRateAlloction[2][0];
-      cfg->ts_target_bitrate[1] = bitrateKbit * 3 / 5;
+      cfg->ts_target_bitrate[1] = bitrateKbit * kVp8LayerRateAlloction[2][1];
       cfg->ts_target_bitrate[2] = bitrateKbit;
       cfg->ts_rate_decimator[0] = 4;
       cfg->ts_rate_decimator[1] = 2;
@@ -107,9 +108,9 @@ bool TemporalLayers::ConfigureBitrates(int bitrateKbit,
       // Bitrate API for VP8 is the agregated bitrate for all lower layers.
       cfg->ts_number_layers = 4;
       cfg->ts_periodicity = temporal_ids_length_;
-      cfg->ts_target_bitrate[0] = bitrateKbit / 4;
+      cfg->ts_target_bitrate[0] = bitrateKbit * kVp8LayerRateAlloction[3][0];
-      cfg->ts_target_bitrate[1] = bitrateKbit * 2 / 5;
+      cfg->ts_target_bitrate[1] = bitrateKbit * kVp8LayerRateAlloction[3][1];
-      cfg->ts_target_bitrate[2] = bitrateKbit * 3 / 5;
+      cfg->ts_target_bitrate[2] = bitrateKbit * kVp8LayerRateAlloction[3][2];
       cfg->ts_target_bitrate[3] = bitrateKbit;
       cfg->ts_rate_decimator[0] = 8;
       cfg->ts_rate_decimator[1] = 4;

src/modules/video_coding/codecs/vp8/main/source/vp8.gypi

@@ -53,6 +53,7 @@
         'reference_picture_selection.h',
         'reference_picture_selection.cc',
         '../interface/vp8.h',
+        '../interface/vp8_common_types.h',
         '../interface/vp8_simulcast.h',
         'vp8.cc',
         'vp8_simulcast.cc',

src/modules/video_coding/main/source/media_opt_util.cc

@@ -8,17 +8,21 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-#include "video_coding_defines.h"
+#include "modules/video_coding/main/source/media_opt_util.h"
-#include "fec_tables_xor.h"
-#include "er_tables_xor.h"
-#include "nack_fec_tables.h"
-#include "qm_select_data.h"
-#include "media_opt_util.h"
 
 #include <math.h>
 #include <float.h>
 #include <limits.h>
 
+#include "modules/interface/module_common_types.h"
+#include "modules/video_coding/codecs/vp8/main/interface/vp8_common_types.h"
+#include "modules/video_coding/main/interface/video_coding_defines.h"
+#include "modules/video_coding/main/source/er_tables_xor.h"
+#include "modules/video_coding/main/source/fec_tables_xor.h"
+#include "modules/video_coding/main/source/nack_fec_tables.h"
+#include "modules/video_coding/main/source/qm_select_data.h"
+
+
 namespace webrtc {
 
 VCMProtectionMethod::VCMProtectionMethod():
@@ -345,12 +349,8 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters)
     // Use a smaller exponent (< 1) to control/soften system size effect.
     const float resolnFac = 1.0 / powf(spatialSizeToRef, 0.3f);
 
-    const float bitRate = parameters->bitRate;
+    const int bitRatePerFrame = BitsPerFrame(parameters);
-    const float frameRate = parameters->frameRate;
 
-    // Average bits per frame (units of kbits)
-    const WebRtc_UWord16 bitRatePerFrame = static_cast<WebRtc_UWord16>
-                                           (bitRate / frameRate);
 
     // Average number of packets per frame (source and fec):
     const WebRtc_UWord8 avgTotPackets = 1 + (WebRtc_UWord8)
@@ -399,11 +399,17 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters)
         codeRateDelta = kPacketLossMax - 1;
     }
 
-    float adjustFec = _qmRobustness->AdjustFecFactor(codeRateDelta,
+    float adjustFec = 1.0f;
-                                                     bitRate,
+    // Avoid additional adjustments when layers are active.
-                                                     frameRate,
+    // TODO(mikhal/marco): Update adjusmtent based on layer info.
-                                                     parameters->rtt,
+    if (parameters->numLayers == 1)
-                                                     packetLoss);
+    {
+        adjustFec = _qmRobustness->AdjustFecFactor(codeRateDelta,
+                                                   parameters->bitRate,
+                                                   parameters->frameRate,
+                                                   parameters->rtt,
+                                                   packetLoss);
+    }
 
     codeRateDelta = static_cast<WebRtc_UWord8>(codeRateDelta * adjustFec);
 
@@ -481,16 +487,36 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters)
     }
 
      // TODO (marpan): Set the UEP protection on/off for Key and Delta frames
-    _useUepProtectionK = _qmRobustness->SetUepProtection(codeRateKey, bitRate,
+    _useUepProtectionK = _qmRobustness->SetUepProtection(codeRateKey,
-                                                         packetLoss, 0);
+                                                         parameters->bitRate,
+                                                         packetLoss,
+                                                         0);
 
-    _useUepProtectionD = _qmRobustness->SetUepProtection(codeRateDelta, bitRate,
+    _useUepProtectionD = _qmRobustness->SetUepProtection(codeRateDelta,
-                                                         packetLoss, 1);
+                                                         parameters->bitRate,
+                                                         packetLoss,
+                                                         1);
 
     // DONE WITH FEC PROTECTION SETTINGS
     return true;
 }
 
+int VCMFecMethod::BitsPerFrame(const VCMProtectionParameters* parameters) {
+  // When temporal layers are available FEC will only be applied on the base
+  // layer.
+  const float bitRateRatio =
+    kVp8LayerRateAlloction[parameters->numLayers - 1][0];
+  float frameRateRatio = powf(1 / 2, parameters->numLayers - 1);
+  float bitRate = parameters->bitRate * bitRateRatio;
+  float frameRate = parameters->frameRate * frameRateRatio;
+
+  // TODO(mikhal): Update factor following testing.
+  float adjustmentFactor = 1;
+
+  // Average bits per frame (units of kbits)
+  return static_cast<int>(adjustmentFactor * bitRate / frameRate);
+}
+
 bool
 VCMFecMethod::EffectivePacketLoss(const VCMProtectionParameters* parameters)
 {
@@ -570,7 +596,8 @@ _packetsPerFrameKey(0.9999f),
 _residualPacketLossFec(0),
 _boostRateKey(2),
 _codecWidth(0),
-_codecHeight(0)
+_codecHeight(0),
+_numLayers(1)
 {
     Reset();
 }
@@ -801,6 +828,10 @@ VCMLossProtectionLogic::UpdateFrameSize(WebRtc_UWord16 width,
     _codecHeight = height;
 }
 
+void VCMLossProtectionLogic::UpdateNumLayers(int numLayers) {
+  _numLayers = (numLayers == 0) ? 1 : numLayers;
+}
+
 bool
 VCMLossProtectionLogic::UpdateMethod()
 {
@@ -820,6 +851,7 @@ VCMLossProtectionLogic::UpdateMethod()
     _currentParameters.residualPacketLossFec = _residualPacketLossFec;
     _currentParameters.codecWidth = _codecWidth;
     _currentParameters.codecHeight = _codecHeight;
+    _currentParameters.numLayers = _numLayers;
     return _selectedMethod->UpdateParameters(&_currentParameters);
 }
 

src/modules/video_coding/main/source/media_opt_util.h

@@ -46,7 +46,8 @@ struct VCMProtectionParameters
     VCMProtectionParameters() : rtt(0), lossPr(0.0f), bitRate(0.0f),
         packetsPerFrame(0.0f), packetsPerFrameKey(0.0f), frameRate(0.0f),
         keyFrameSize(0.0f), fecRateDelta(0), fecRateKey(0),
-        residualPacketLossFec(0.0f), codecWidth(0), codecHeight(0)
+        residualPacketLossFec(0.0f), codecWidth(0), codecHeight(0),
+        numLayers(1)
         {}
 
     int                 rtt;
@@ -61,6 +62,7 @@ struct VCMProtectionParameters
     float               residualPacketLossFec;
     WebRtc_UWord16      codecWidth;
     WebRtc_UWord16      codecHeight;
+    int                 numLayers;
 };
 
 
@@ -189,6 +191,8 @@ public:
     void UpdateProtectionFactorD(WebRtc_UWord8 protectionFactorD);
     // Update FEC with protectionFactorK
     void UpdateProtectionFactorK(WebRtc_UWord8 protectionFactorK);
+    // Compute the bits per frame. Account for temporal layers when applicable.
+    int BitsPerFrame(const VCMProtectionParameters* parameters);
 };
 
 
@@ -297,6 +301,12 @@ public:
     //          - height       : The codec frame height.
     void UpdateFrameSize(WebRtc_UWord16 width, WebRtc_UWord16 height);
 
+    // Update the number of active layers
+    //
+    // Input:
+    //          - numLayers    : Number of layers used.
+    void UpdateNumLayers(int numLayers);
+
     // The amount of packet loss to cover for with FEC.
     //
     // Input:
@@ -355,6 +365,7 @@ private:
     WebRtc_UWord8             _boostRateKey;
     WebRtc_UWord16            _codecWidth;
     WebRtc_UWord16            _codecHeight;
+    int                       _numLayers;
 };
 
 } // namespace webrtc

src/modules/video_coding/main/source/media_optimization.cc

@@ -36,7 +36,8 @@ _avgSentBitRateBps(0.0f),
 _keyFrameCnt(0),
 _deltaFrameCnt(0),
 _lastQMUpdateTime(0),
-_lastChangeTime(0)
+_lastChangeTime(0),
+_numLayers(0)
 {
     memset(_sendStatistics, 0, sizeof(_sendStatistics));
     memset(_incomingFrameTimes, -1, sizeof(_incomingFrameTimes));
@@ -83,6 +84,7 @@ VCMMediaOptimization::Reset()
         _encodedFrameSamples[i]._timeCompleteMs = -1;
     }
     _avgSentBitRateBps = 0.0f;
+    _numLayers = 1;
     return VCM_OK;
 }
 
@@ -170,7 +172,7 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
     // Update encoding rates following protection settings
     _frameDropper->SetRates(static_cast<float>(_targetBitRate), 0);
 
-    if (_enableQm)
+    if (_enableQm && _numLayers == 1)
     {
         // Update QM with rates
         _qmResolution->UpdateRates((float)_targetBitRate, _avgSentBitRateBps,
@@ -218,6 +220,8 @@ int VCMMediaOptimization::UpdateProtectionCallback(
     bool nackStatus = (selected_method->Type() == kNackFec ||
                        selected_method->Type() == kNack);
 
+    // TODO(Marco): Pass FEC protection values per layer.
+
     return _videoProtectionCallback->ProtectionRequest(codeRateDeltaRTP,
                                                        codeRateKeyRTP,
                                                        useUepProtectionDeltaRTP,
@@ -245,9 +249,13 @@ VCMMediaOptimization::SentFrameCount(VCMFrameCount &frameCount) const
 }
 
 WebRtc_Word32
-VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, WebRtc_Word32 maxBitRate,
+VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType,
-                                      WebRtc_UWord32 frameRate, WebRtc_UWord32 bitRate,
+                                      WebRtc_Word32 maxBitRate,
-                                      WebRtc_UWord16 width, WebRtc_UWord16 height)
+                                      WebRtc_UWord32 frameRate,
+                                      WebRtc_UWord32 bitRate,
+                                      WebRtc_UWord16 width,
+                                      WebRtc_UWord16 height,
+                                      int numLayers)
 {
     // Everything codec specific should be reset here since this means the codec
     // has changed. If native dimension values have changed, then either user
@@ -263,12 +271,14 @@ VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, WebRtc_Word3
     _lossProtLogic->UpdateBitRate(static_cast<float>(bitRate));
     _lossProtLogic->UpdateFrameRate(static_cast<float>(frameRate));
     _lossProtLogic->UpdateFrameSize(width, height);
+    _lossProtLogic->UpdateNumLayers(numLayers);
     _frameDropper->Reset();
     _frameDropper->SetRates(static_cast<float>(bitRate),
                             static_cast<float>(frameRate));
     _userFrameRate = static_cast<float>(frameRate);
     _codecWidth = width;
     _codecHeight = height;
+    _numLayers = (numLayers <= 1) ? 1 : numLayers;  // Can also be zero.
     WebRtc_Word32 ret = VCM_OK;
     ret = _qmResolution->Initialize((float)_targetBitRate, _userFrameRate,
                                     _codecWidth, _codecHeight);

src/modules/video_coding/main/source/media_optimization.h

@@ -65,7 +65,8 @@ public:
                                   WebRtc_UWord32 frameRate,
                                   WebRtc_UWord32 bitRate,
                                   WebRtc_UWord16 width,
-                                  WebRtc_UWord16 height);
+                                  WebRtc_UWord16 height,
+                                  int numTemporalLayers);
     /**
     * Enable protection method
     */
@@ -199,6 +200,7 @@ private:
 
     WebRtc_Word64                     _lastQMUpdateTime;
     WebRtc_Word64                     _lastChangeTime; // content/user triggered
+    int                               _numLayers;
 
 
 }; // end of VCMMediaOptimization class definition

src/modules/video_coding/main/source/video_coding_impl.cc

@@ -366,7 +366,7 @@ VideoCodingModuleImpl::InitializeSender()
     _encoder = NULL;
     _encodedFrameCallback.SetTransportCallback(NULL);
     // setting default bitRate and frameRate to 0
-    _mediaOpt.SetEncodingData(kVideoCodecUnknown, 0, 0, 0, 0, 0);
+    _mediaOpt.SetEncodingData(kVideoCodecUnknown, 0, 0, 0, 0, 0, 0);
     _mediaOpt.Reset(); // Resetting frame dropper
     return VCM_OK;
 }
@@ -420,12 +420,16 @@ VideoCodingModuleImpl::RegisterSendCodec(const VideoCodec* sendCodec,
         return VCM_CODEC_ERROR;
     }
     _sendCodecType = sendCodec->codecType;
+    int numLayers = (_sendCodecType != kVideoCodecVP8) ? 1 :
+                        sendCodec->codecSpecific.VP8.numberOfTemporalLayers;
+
     _mediaOpt.SetEncodingData(_sendCodecType,
                               sendCodec->maxBitrate,
                               sendCodec->maxFramerate,
                               sendCodec->startBitrate,
                               sendCodec->width,
-                              sendCodec->height);
+                              sendCodec->height,
+                              numLayers);
     _mediaOpt.SetMtu(maxPayloadSize);
 
     return VCM_OK;