generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Adding layer info consideration when applying FEC protection. In this first version, we hard code protection zero for non-base layer frames. As a future enhancement, an array should be passed from mediaOpt to set the protection per layer. A TODO was added in MediaOpt.

Browse Source 
Review URL: http://webrtc-codereview.appspot.com/330005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@1238 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
mikhal@webrtc.org 2011-12-19 19:04:49 +00:00
parent 190e88a6d3
commit 0e7d9d862a
9 changed files with 133 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/modules/rtp_rtcp/source/rtp_sender_video.cc
View File

@ -436,6 +436,11 @@ RTPSenderVideo::SendVideo(const RtpVideoCodecTypes videoType,
{
_fecProtectionFactor = _codeRateKey;
_fecUseUepProtection = _useUepProtectionKey;
} else if (videoType == kRtpVp8Video && rtpTypeHdr->VP8.temporalIdx > 0)
{
// In current version, we only apply FEC on the base layer.
_fecProtectionFactor = 0;
_fecUseUepProtection = false;
} else
{
_fecProtectionFactor = _codeRateDelta;

29
src/modules/video_coding/codecs/vp8/main/interface/vp8_common_types.h Normal file
View File

@ -0,0 +1,29 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_COMMON_TYPES_H_
#define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_COMMON_TYPES_H_
#include "common_types.h"
namespace webrtc {
// Ratio allocation between temporal streams:
// Values as required for the VP8 codec (accumulating).
static const float
kVp8LayerRateAlloction[kMaxTemporalStreams][kMaxTemporalStreams] = {
{1.0f, 0, 0, 0}, // 1 layer
{0.6f, 1.0f , 0 , 0}, // 2 layers {60%, 40%}
{0.4f, 0.6f , 1.0f, 0}, // 3 layers {40%, 20%, 40%}
{0.25f, 0.4f, 0.6f, 1.0f} // 4 layers {25%, 15%, 20%, 40%}
};
} // namespace webrtc
#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_COMMON_TYPES_H_

17
src/modules/video_coding/codecs/vp8/main/source/temporal_layers.cc
View File

@ -13,8 +13,9 @@
#include <string.h>
#include <cassert>
#include "module_common_types.h"
#include "video_codec_interface.h"
#include "modules/interface/module_common_types.h"
#include "modules/video_coding/codecs/interface/video_codec_interface.h"
#include "modules/video_coding/codecs/vp8/main/interface/vp8_common_types.h"
#include "vpx/vpx_encoder.h"
#include "vpx/vp8cx.h"
@ -47,7 +48,7 @@ bool TemporalLayers::ConfigureBitrates(int bitrateKbit,
cfg->ts_periodicity = temporal_ids_length_;
// Split stream 60% 40%.
// Bitrate API for VP8 is the agregated bitrate for all lower layers.
cfg->ts_target_bitrate[0] = bitrateKbit * 3 / 5;
cfg->ts_target_bitrate[0] = bitrateKbit * kVp8LayerRateAlloction[1][0];
cfg->ts_target_bitrate[1] = bitrateKbit;
cfg->ts_rate_decimator[0] = 2;
cfg->ts_rate_decimator[1] = 1;
@ -74,8 +75,8 @@ bool TemporalLayers::ConfigureBitrates(int bitrateKbit,
cfg->ts_periodicity = temporal_ids_length_;
// Split stream 40% 20% 40%.
// Bitrate API for VP8 is the agregated bitrate for all lower layers.
cfg->ts_target_bitrate[0] = bitrateKbit * 2 / 5;
cfg->ts_target_bitrate[1] = bitrateKbit * 3 / 5;
cfg->ts_target_bitrate[0] = bitrateKbit * kVp8LayerRateAlloction[2][0];
cfg->ts_target_bitrate[1] = bitrateKbit * kVp8LayerRateAlloction[2][1];
cfg->ts_target_bitrate[2] = bitrateKbit;
cfg->ts_rate_decimator[0] = 4;
cfg->ts_rate_decimator[1] = 2;
@ -107,9 +108,9 @@ bool TemporalLayers::ConfigureBitrates(int bitrateKbit,
// Bitrate API for VP8 is the agregated bitrate for all lower layers.
cfg->ts_number_layers = 4;
cfg->ts_periodicity = temporal_ids_length_;
cfg->ts_target_bitrate[0] = bitrateKbit / 4;
cfg->ts_target_bitrate[1] = bitrateKbit * 2 / 5;
cfg->ts_target_bitrate[2] = bitrateKbit * 3 / 5;
cfg->ts_target_bitrate[0] = bitrateKbit * kVp8LayerRateAlloction[3][0];
cfg->ts_target_bitrate[1] = bitrateKbit * kVp8LayerRateAlloction[3][1];
cfg->ts_target_bitrate[2] = bitrateKbit * kVp8LayerRateAlloction[3][2];
cfg->ts_target_bitrate[3] = bitrateKbit;
cfg->ts_rate_decimator[0] = 8;
cfg->ts_rate_decimator[1] = 4;

1
src/modules/video_coding/codecs/vp8/main/source/vp8.gypi
View File

@ -53,6 +53,7 @@
'reference_picture_selection.h',
'reference_picture_selection.cc',
'../interface/vp8.h',
'../interface/vp8_common_types.h',
'../interface/vp8_simulcast.h',
'vp8.cc',
'vp8_simulcast.cc',

70
src/modules/video_coding/main/source/media_opt_util.cc
View File

@ -8,17 +8,21 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video_coding_defines.h"
#include "fec_tables_xor.h"
#include "er_tables_xor.h"
#include "nack_fec_tables.h"
#include "qm_select_data.h"
#include "media_opt_util.h"
#include "modules/video_coding/main/source/media_opt_util.h"
#include <math.h>
#include <float.h>
#include <limits.h>
#include "modules/interface/module_common_types.h"
#include "modules/video_coding/codecs/vp8/main/interface/vp8_common_types.h"
#include "modules/video_coding/main/interface/video_coding_defines.h"
#include "modules/video_coding/main/source/er_tables_xor.h"
#include "modules/video_coding/main/source/fec_tables_xor.h"
#include "modules/video_coding/main/source/nack_fec_tables.h"
#include "modules/video_coding/main/source/qm_select_data.h"
namespace webrtc {
VCMProtectionMethod::VCMProtectionMethod():
@ -345,12 +349,8 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters)
// Use a smaller exponent (< 1) to control/soften system size effect.
const float resolnFac = 1.0 / powf(spatialSizeToRef, 0.3f);
const float bitRate = parameters->bitRate;
const float frameRate = parameters->frameRate;
const int bitRatePerFrame = BitsPerFrame(parameters);
// Average bits per frame (units of kbits)
const WebRtc_UWord16 bitRatePerFrame = static_cast<WebRtc_UWord16>
(bitRate / frameRate);
// Average number of packets per frame (source and fec):
const WebRtc_UWord8 avgTotPackets = 1 + (WebRtc_UWord8)
@ -399,11 +399,17 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters)
codeRateDelta = kPacketLossMax - 1;
}
float adjustFec = _qmRobustness->AdjustFecFactor(codeRateDelta,
bitRate,
frameRate,
float adjustFec = 1.0f;
// Avoid additional adjustments when layers are active.
// TODO(mikhal/marco): Update adjusmtent based on layer info.
if (parameters->numLayers == 1)
{
adjustFec = _qmRobustness->AdjustFecFactor(codeRateDelta,
parameters->bitRate,
parameters->frameRate,
parameters->rtt,
packetLoss);
}
codeRateDelta = static_cast<WebRtc_UWord8>(codeRateDelta * adjustFec);
@ -481,16 +487,36 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters)
}
// TODO (marpan): Set the UEP protection on/off for Key and Delta frames
_useUepProtectionK = _qmRobustness->SetUepProtection(codeRateKey, bitRate,
packetLoss, 0);
_useUepProtectionK = _qmRobustness->SetUepProtection(codeRateKey,
parameters->bitRate,
packetLoss,
0);
_useUepProtectionD = _qmRobustness->SetUepProtection(codeRateDelta, bitRate,
packetLoss, 1);
_useUepProtectionD = _qmRobustness->SetUepProtection(codeRateDelta,
parameters->bitRate,
packetLoss,
1);
// DONE WITH FEC PROTECTION SETTINGS
return true;
}
int VCMFecMethod::BitsPerFrame(const VCMProtectionParameters* parameters) {
// When temporal layers are available FEC will only be applied on the base
// layer.
const float bitRateRatio =
kVp8LayerRateAlloction[parameters->numLayers - 1][0];
float frameRateRatio = powf(1 / 2, parameters->numLayers - 1);
float bitRate = parameters->bitRate * bitRateRatio;
float frameRate = parameters->frameRate * frameRateRatio;
// TODO(mikhal): Update factor following testing.
float adjustmentFactor = 1;
// Average bits per frame (units of kbits)
return static_cast<int>(adjustmentFactor * bitRate / frameRate);
}
bool
VCMFecMethod::EffectivePacketLoss(const VCMProtectionParameters* parameters)
{
@ -570,7 +596,8 @@ _packetsPerFrameKey(0.9999f),
_residualPacketLossFec(0),
_boostRateKey(2),
_codecWidth(0),
_codecHeight(0)
_codecHeight(0),
_numLayers(1)
{
Reset();
}
@ -801,6 +828,10 @@ VCMLossProtectionLogic::UpdateFrameSize(WebRtc_UWord16 width,
_codecHeight = height;
}
void VCMLossProtectionLogic::UpdateNumLayers(int numLayers) {
_numLayers = (numLayers == 0) ? 1 : numLayers;
}
bool
VCMLossProtectionLogic::UpdateMethod()
{
@ -820,6 +851,7 @@ VCMLossProtectionLogic::UpdateMethod()
_currentParameters.residualPacketLossFec = _residualPacketLossFec;
_currentParameters.codecWidth = _codecWidth;
_currentParameters.codecHeight = _codecHeight;
_currentParameters.numLayers = _numLayers;
return _selectedMethod->UpdateParameters(&_currentParameters);
}

13
src/modules/video_coding/main/source/media_opt_util.h
View File

@ -46,7 +46,8 @@ struct VCMProtectionParameters
VCMProtectionParameters() : rtt(0), lossPr(0.0f), bitRate(0.0f),
packetsPerFrame(0.0f), packetsPerFrameKey(0.0f), frameRate(0.0f),
keyFrameSize(0.0f), fecRateDelta(0), fecRateKey(0),
residualPacketLossFec(0.0f), codecWidth(0), codecHeight(0)
residualPacketLossFec(0.0f), codecWidth(0), codecHeight(0),
numLayers(1)
{}
int rtt;
@ -61,6 +62,7 @@ struct VCMProtectionParameters
float residualPacketLossFec;
WebRtc_UWord16 codecWidth;
WebRtc_UWord16 codecHeight;
int numLayers;
};
@ -189,6 +191,8 @@ public:
void UpdateProtectionFactorD(WebRtc_UWord8 protectionFactorD);
// Update FEC with protectionFactorK
void UpdateProtectionFactorK(WebRtc_UWord8 protectionFactorK);
// Compute the bits per frame. Account for temporal layers when applicable.
int BitsPerFrame(const VCMProtectionParameters* parameters);
};
@ -297,6 +301,12 @@ public:
// - height : The codec frame height.
void UpdateFrameSize(WebRtc_UWord16 width, WebRtc_UWord16 height);
// Update the number of active layers
//
// Input:
// - numLayers : Number of layers used.
void UpdateNumLayers(int numLayers);
// The amount of packet loss to cover for with FEC.
//
// Input:
@ -355,6 +365,7 @@ private:
WebRtc_UWord8 _boostRateKey;
WebRtc_UWord16 _codecWidth;
WebRtc_UWord16 _codecHeight;
int _numLayers;
};
} // namespace webrtc

20
src/modules/video_coding/main/source/media_optimization.cc
View File

@ -36,7 +36,8 @@ _avgSentBitRateBps(0.0f),
_keyFrameCnt(0),
_deltaFrameCnt(0),
_lastQMUpdateTime(0),
_lastChangeTime(0)
_lastChangeTime(0),
_numLayers(0)
{
memset(_sendStatistics, 0, sizeof(_sendStatistics));
memset(_incomingFrameTimes, -1, sizeof(_incomingFrameTimes));
@ -83,6 +84,7 @@ VCMMediaOptimization::Reset()
_encodedFrameSamples[i]._timeCompleteMs = -1;
}
_avgSentBitRateBps = 0.0f;
_numLayers = 1;
return VCM_OK;
}
@ -170,7 +172,7 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
// Update encoding rates following protection settings
_frameDropper->SetRates(static_cast<float>(_targetBitRate), 0);
if (_enableQm)
if (_enableQm && _numLayers == 1)
{
// Update QM with rates
_qmResolution->UpdateRates((float)_targetBitRate, _avgSentBitRateBps,
@ -218,6 +220,8 @@ int VCMMediaOptimization::UpdateProtectionCallback(
bool nackStatus = (selected_method->Type() == kNackFec ||
selected_method->Type() == kNack);
// TODO(Marco): Pass FEC protection values per layer.
return _videoProtectionCallback->ProtectionRequest(codeRateDeltaRTP,
codeRateKeyRTP,
useUepProtectionDeltaRTP,
@ -245,9 +249,13 @@ VCMMediaOptimization::SentFrameCount(VCMFrameCount &frameCount) const
}
WebRtc_Word32
VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, WebRtc_Word32 maxBitRate,
WebRtc_UWord32 frameRate, WebRtc_UWord32 bitRate,
WebRtc_UWord16 width, WebRtc_UWord16 height)
VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType,
WebRtc_Word32 maxBitRate,
WebRtc_UWord32 frameRate,
WebRtc_UWord32 bitRate,
WebRtc_UWord16 width,
WebRtc_UWord16 height,
int numLayers)
{
// Everything codec specific should be reset here since this means the codec
// has changed. If native dimension values have changed, then either user
@ -263,12 +271,14 @@ VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, WebRtc_Word3
_lossProtLogic->UpdateBitRate(static_cast<float>(bitRate));
_lossProtLogic->UpdateFrameRate(static_cast<float>(frameRate));
_lossProtLogic->UpdateFrameSize(width, height);
_lossProtLogic->UpdateNumLayers(numLayers);
_frameDropper->Reset();
_frameDropper->SetRates(static_cast<float>(bitRate),
static_cast<float>(frameRate));
_userFrameRate = static_cast<float>(frameRate);
_codecWidth = width;
_codecHeight = height;
_numLayers = (numLayers <= 1) ? 1 : numLayers; // Can also be zero.
WebRtc_Word32 ret = VCM_OK;
ret = _qmResolution->Initialize((float)_targetBitRate, _userFrameRate,
_codecWidth, _codecHeight);

4
src/modules/video_coding/main/source/media_optimization.h
View File

@ -65,7 +65,8 @@ public:
WebRtc_UWord32 frameRate,
WebRtc_UWord32 bitRate,
WebRtc_UWord16 width,
WebRtc_UWord16 height);
WebRtc_UWord16 height,
int numTemporalLayers);
/**
* Enable protection method
*/
@ -199,6 +200,7 @@ private:
WebRtc_Word64 _lastQMUpdateTime;
WebRtc_Word64 _lastChangeTime; // content/user triggered
int _numLayers;
}; // end of VCMMediaOptimization class definition

8
src/modules/video_coding/main/source/video_coding_impl.cc
View File

@ -366,7 +366,7 @@ VideoCodingModuleImpl::InitializeSender()
_encoder = NULL;
_encodedFrameCallback.SetTransportCallback(NULL);
// setting default bitRate and frameRate to 0
_mediaOpt.SetEncodingData(kVideoCodecUnknown, 0, 0, 0, 0, 0);
_mediaOpt.SetEncodingData(kVideoCodecUnknown, 0, 0, 0, 0, 0, 0);
_mediaOpt.Reset(); // Resetting frame dropper
return VCM_OK;
}
@ -420,12 +420,16 @@ VideoCodingModuleImpl::RegisterSendCodec(const VideoCodec* sendCodec,
return VCM_CODEC_ERROR;
}
_sendCodecType = sendCodec->codecType;
int numLayers = (_sendCodecType != kVideoCodecVP8) ? 1 :
sendCodec->codecSpecific.VP8.numberOfTemporalLayers;
_mediaOpt.SetEncodingData(_sendCodecType,
sendCodec->maxBitrate,
sendCodec->maxFramerate,
sendCodec->startBitrate,
sendCodec->width,
sendCodec->height);
sendCodec->height,
numLayers);
_mediaOpt.SetMtu(maxPayloadSize);
return VCM_OK;