generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Modified media_opt and qm_select to allow for robustness settings

Browse Source 
based on the content metrics. Re-organized the class structure in qm_select
into a resolution class and robustness settings class, both derived from
a main (qm/content analysis) class.
Review URL: http://webrtc-codereview.appspot.com/55006

git-svn-id: http://webrtc.googlecode.com/svn/trunk@188 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
marpan@google.com 2011-07-12 17:12:57 +00:00
parent 6bed064e2f
commit 86548c62e9
7 changed files with 425 additions and 311 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/modules/video_coding/main/source/media_opt_util.cc
View File

@ -21,6 +21,13 @@
namespace webrtc { namespace webrtc {
void
VCMProtectionMethod::UpdateContentMetrics(
const VideoContentMetrics* contentMetrics)
{
_qmRobustness->UpdateContent(contentMetrics);
}
bool bool
VCMProtectionMethod::BetterThan(VCMProtectionMethod *pm) VCMProtectionMethod::BetterThan(VCMProtectionMethod *pm)
{ {
@ -417,6 +424,10 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters)
codeRateDelta = plossMax - 1; codeRateDelta = plossMax - 1;
} }
codeRateDelta = _qmRobustness->AdjustFecFactor(codeRateDelta, bitRate,
parameters->frameRate,
parameters->rtt, packetLoss);
// For Key frame: // For Key frame:
// Effectively at a higher rate, so we scale/boost the rate // Effectively at a higher rate, so we scale/boost the rate
// The boost factor may depend on several factors: ratio of packet // The boost factor may depend on several factors: ratio of packet
@ -462,6 +473,14 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters)
_protectionFactorK = codeRateKey; _protectionFactorK = codeRateKey;
_protectionFactorD = codeRateDelta; _protectionFactorD = codeRateDelta;
// Set the UEP protection on/off for Key and Delta frames
_uepKey = _qmRobustness->SetUepProtection(codeRateKey, bitRate,
packetLoss, 0);
_uepDelta = _qmRobustness->SetUepProtection(codeRateKey, bitRate,
packetLoss, 1);
// DONE WITH FEC PROTECTION SETTINGS // DONE WITH FEC PROTECTION SETTINGS
return true; return true;
} }

14
src/modules/video_coding/main/source/media_opt_util.h
View File

@ -17,6 +17,7 @@
#include "exp_filter.h" #include "exp_filter.h"
#include "internal_defines.h" #include "internal_defines.h"
#include "tick_time.h" #include "tick_time.h"
#include "qm_select.h"
#include <cmath> #include <cmath>
#include <cstdlib> #include <cstdlib>
@ -97,8 +98,10 @@ public:
//friend VCMProtectionMethod; //friend VCMProtectionMethod;
VCMProtectionMethod(VCMProtectionMethodEnum type) : _protectionFactorK(0), VCMProtectionMethod(VCMProtectionMethodEnum type) : _protectionFactorK(0),
_protectionFactorD(0), _residualPacketLoss(0.0), _scaleProtKey(2.0), _protectionFactorD(0), _residualPacketLoss(0.0), _scaleProtKey(2.0),
_maxPayloadSize(1460), _efficiency(0), _score(0), _type(type) {} _maxPayloadSize(1460), _efficiency(0), _score(0), _type(type),
virtual ~VCMProtectionMethod() {} _uepKey(0), _uepDelta(1)
{_qmRobustness = new VCMQmRobustness();}
virtual ~VCMProtectionMethod() { delete _qmRobustness;}
// Updates the efficiency of the method using the parameters provided // Updates the efficiency of the method using the parameters provided
// //
@ -142,6 +145,9 @@ public:
// Return value : Required protectionFactor for delta frame // Return value : Required protectionFactor for delta frame
virtual WebRtc_UWord8 RequiredProtectionFactorD() { return _protectionFactorD; } virtual WebRtc_UWord8 RequiredProtectionFactorD() { return _protectionFactorD; }
// Updates content metrics
void UpdateContentMetrics(const VideoContentMetrics* contentMetrics);
WebRtc_UWord8 _effectivePacketLoss; WebRtc_UWord8 _effectivePacketLoss;
WebRtc_UWord8 _protectionFactorK; WebRtc_UWord8 _protectionFactorK;
WebRtc_UWord8 _protectionFactorD; WebRtc_UWord8 _protectionFactorD;
@ -149,6 +155,10 @@ public:
float _scaleProtKey; float _scaleProtKey;
WebRtc_Word32 _maxPayloadSize; WebRtc_Word32 _maxPayloadSize;
VCMQmRobustness* _qmRobustness;
bool _uepKey;
bool _uepDelta;
protected: protected:
float _efficiency; float _efficiency;
float _score; float _score;

40
src/modules/video_coding/main/source/media_optimization.cc
View File

@ -46,7 +46,7 @@ _lastChangeTime(0)
_frameDropper = new VCMFrameDropper(_id); _frameDropper = new VCMFrameDropper(_id);
_lossProtLogic = new VCMLossProtectionLogic(); _lossProtLogic = new VCMLossProtectionLogic();
_content = new VCMContentMetricsProcessing(); _content = new VCMContentMetricsProcessing();
_qms = new VCMQmSelect(); _qmResolution = new VCMQmResolution();
} }
VCMMediaOptimization::~VCMMediaOptimization(void) VCMMediaOptimization::~VCMMediaOptimization(void)
@ -55,7 +55,7 @@ VCMMediaOptimization::~VCMMediaOptimization(void)
delete _lossProtLogic; delete _lossProtLogic;
delete _frameDropper; delete _frameDropper;
delete _content; delete _content;
delete _qms; delete _qmResolution;
} }
WebRtc_Word32 WebRtc_Word32
@ -67,7 +67,7 @@ VCMMediaOptimization::Reset()
_lossProtLogic->Reset(); _lossProtLogic->Reset();
_frameDropper->SetRates(0, 0); _frameDropper->SetRates(0, 0);
_content->Reset(); _content->Reset();
_qms->Reset(); _qmResolution->Reset();
_lossProtLogic->UpdateFrameRate(_incomingFrameRate); _lossProtLogic->UpdateFrameRate(_incomingFrameRate);
_lossProtLogic->Reset(); _lossProtLogic->Reset();
_sendStatisticsZeroEncode = 0; _sendStatisticsZeroEncode = 0;
@ -135,6 +135,10 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
selectedMethod->Type() == kNackFec )) selectedMethod->Type() == kNackFec ))
{ {
// Update protection method with content metrics
selectedMethod->UpdateContentMetrics(_content->ShortTermAvgData());
// Update method will compute the robustness settings for the given // Update method will compute the robustness settings for the given
// protection method and the overhead cost // protection method and the overhead cost
// the protection method is set by the user via SetVideoProtection. // the protection method is set by the user via SetVideoProtection.
@ -197,8 +201,8 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
if (_enableQm) if (_enableQm)
{ {
//Update QM with rates //Update QM with rates
_qms->UpdateRates((float)_targetBitRate, _avgSentBitRateBps, _qmResolution->UpdateRates((float)_targetBitRate, _avgSentBitRateBps,
_incomingFrameRate, _fractionLost); _incomingFrameRate, _fractionLost);
//Check for QM selection //Check for QM selection
bool selectQM = checkStatusForQMchange(); bool selectQM = checkStatusForQMchange();
if (selectQM) if (selectQM)
@ -249,11 +253,12 @@ VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, WebRtc_Word3
_lossProtLogic->UpdateFrameSize(width, height); _lossProtLogic->UpdateFrameSize(width, height);
_frameDropper->Reset(); _frameDropper->Reset();
_frameDropper->SetRates(static_cast<float>(bitRate), static_cast<float>(frameRate)); _frameDropper->SetRates(static_cast<float>(bitRate), static_cast<float>(frameRate));
_userFrameRate = (float)frameRate; _userFrameRate = static_cast<float>(frameRate);
_codecWidth = width; _codecWidth = width;
_codecHeight = height; _codecHeight = height;
WebRtc_Word32 ret = VCM_OK; WebRtc_Word32 ret = VCM_OK;
ret = _qms->Initialize((float)_targetBitRate, _userFrameRate, _codecWidth, _codecHeight); ret = _qmResolution->Initialize((float)_targetBitRate, _userFrameRate,
_codecWidth, _codecHeight);
return ret; return ret;
} }
@ -420,7 +425,8 @@ VCMMediaOptimization::UpdateWithEncodedData(WebRtc_Word32 encodedLength,
if (_enableQm) if (_enableQm)
{ {
// update quality select with encoded length // update quality select with encoded length
_qms->UpdateEncodedSize(encodedLength, encodedFrameType); _qmResolution->UpdateEncodedSize(encodedLength,
encodedFrameType);
} }
} }
if (!deltaFrame && encodedLength > 0) if (!deltaFrame && encodedLength > 0)
@ -525,7 +531,7 @@ VCMMediaOptimization::updateContentData(const VideoContentMetrics *contentMetric
{ {
//No QM if metrics are NULL //No QM if metrics are NULL
_enableQm = false; _enableQm = false;
_qms->Reset(); _qmResolution->Reset();
} }
else else
{ {
@ -537,14 +543,14 @@ WebRtc_Word32
VCMMediaOptimization::SelectQuality() VCMMediaOptimization::SelectQuality()
{ {
// Reset quantities for QM select // Reset quantities for QM select
_qms->ResetQM(); _qmResolution->ResetQM();
// Update QM will long-term averaged content metrics. // Update QM will long-term averaged content metrics.
_qms->UpdateContent(_content->LongTermAvgData()); _qmResolution->UpdateContent(_content->LongTermAvgData());
// Select quality mode // Select quality mode
VCMQualityMode* qm = NULL; VCMResolutionScale* qm = NULL;
WebRtc_Word32 ret = _qms->SelectQuality(&qm); WebRtc_Word32 ret = _qmResolution->SelectResolution(&qm);
if (ret < 0) if (ret < 0)
{ {
return ret; return ret;
@ -554,7 +560,7 @@ VCMMediaOptimization::SelectQuality()
QMUpdate(qm); QMUpdate(qm);
// Reset all the rate and related frame counters quantities // Reset all the rate and related frame counters quantities
_qms->ResetRates(); _qmResolution->ResetRates();
// Reset counters // Reset counters
_lastQMUpdateTime = VCMTickTime::MillisecondTimestamp(); _lastQMUpdateTime = VCMTickTime::MillisecondTimestamp();
@ -592,7 +598,7 @@ VCMMediaOptimization::checkStatusForQMchange()
} }
bool bool
VCMMediaOptimization::QMUpdate(VCMQualityMode* qm) VCMMediaOptimization::QMUpdate(VCMResolutionScale* qm)
{ {
// Check for no change // Check for no change
if (qm->spatialHeightFact == 1 && if (qm->spatialHeightFact == 1 &&
@ -606,7 +612,9 @@ VCMMediaOptimization::QMUpdate(VCMQualityMode* qm)
VideoContentMetrics* cm = _content->LongTermAvgData(); VideoContentMetrics* cm = _content->LongTermAvgData();
// Temporal // Temporal
WebRtc_UWord32 frameRate = static_cast<WebRtc_UWord32>(_incomingFrameRate + 0.5f); WebRtc_UWord32 frameRate = static_cast<WebRtc_UWord32>
(_incomingFrameRate + 0.5f);
// Check if go back up in temporal resolution // Check if go back up in temporal resolution
if (qm->temporalFact == 0) if (qm->temporalFact == 0)
{ {

4
src/modules/video_coding/main/source/media_optimization.h
View File

@ -159,7 +159,7 @@ private:
* verify if QM settings differ from default, i.e. if an update is required * verify if QM settings differ from default, i.e. if an update is required
* Compute actual values, as will be sent to the encoder * Compute actual values, as will be sent to the encoder
*/ */
bool QMUpdate(VCMQualityMode* qm); bool QMUpdate(VCMResolutionScale* qm);
/** /**
* check if we should make a QM change * check if we should make a QM change
* will return 1 if yes, 0 otherwise * will return 1 if yes, 0 otherwise
@ -207,7 +207,7 @@ private:
WebRtc_UWord32 _deltaFrameCnt; WebRtc_UWord32 _deltaFrameCnt;
VCMContentMetricsProcessing* _content; VCMContentMetricsProcessing* _content;
VCMQmSelect* _qms; VCMQmResolution* _qmResolution;
WebRtc_Word64 _lastQMUpdateTime; WebRtc_Word64 _lastQMUpdateTime;
WebRtc_Word64 _lastChangeTime; // content or user triggered WebRtc_Word64 _lastChangeTime; // content or user triggered

473
src/modules/video_coding/main/source/qm_select.cc
View File

@ -20,36 +20,204 @@
namespace webrtc { namespace webrtc {
VCMQmSelect::VCMQmSelect() // QM-METHOD class
VCMQmMethod::VCMQmMethod()
{ {
_qm = new VCMQualityMode();
_contentMetrics = new VideoContentMetrics(); _contentMetrics = new VideoContentMetrics();
Reset(); ResetQM();
} }
VCMQmSelect::~VCMQmSelect() VCMQmMethod::~VCMQmMethod()
{ {
delete _qm;
delete _contentMetrics; delete _contentMetrics;
} }
void void
VCMQmSelect::ResetQM() VCMQmMethod::ResetQM()
{ {
_motion.Reset(); _motion.Reset();
_spatial.Reset(); _spatial.Reset();
_coherence.Reset(); _coherence.Reset();
_stationaryMotion = 0; _stationaryMotion = 0;
_aspectRatio = 1; _aspectRatio = 1;
_maxRateQM = 0; _imageType = 2;
_imageType = 1;
_userResolutionPref = 50; // Neutral
_qm->Reset();
return; return;
} }
void void
VCMQmSelect::ResetRates() VCMQmMethod::UpdateContent(const VideoContentMetrics* contentMetrics)
{
_contentMetrics = contentMetrics;
}
void
VCMQmMethod::MotionNFD()
{
_motion.value = _contentMetrics->motionMagnitudeNZ;
// Determine motion level
if (_motion.value < LOW_MOTION_NFD)
{
_motion.level = kLow;
}
else if (_motion.value > HIGH_MOTION_NFD)
{
_motion.level = kHigh;
}
else
{
_motion.level = kDefault;
}
}
void
VCMQmMethod::Motion()
{
float sizeZeroMotion = _contentMetrics->sizeZeroMotion;
float motionMagNZ = _contentMetrics->motionMagnitudeNZ;
// Take product of size and magnitude with equal weight
_motion.value = (1.0f - sizeZeroMotion) * motionMagNZ;
// Stabilize: motionMagNZ could be large when only a
// few motion blocks are non-zero
_stationaryMotion = false;
if (sizeZeroMotion > HIGH_ZERO_MOTION_SIZE)
{
_motion.value = 0.0f;
_stationaryMotion = true;
}
// Determine motion level
if (_motion.value < LOW_MOTION)
{
_motion.level = kLow;
}
else if (_motion.value > HIGH_MOTION)
{
_motion.level = kHigh;
}
else
{
_motion.level = kDefault;
}
}
void
VCMQmMethod::Spatial()
{
float spatialErr = _contentMetrics->spatialPredErr;
float spatialErrH = _contentMetrics->spatialPredErrH;
float spatialErrV = _contentMetrics->spatialPredErrV;
// Spatial measure: take average of 3 prediction errors
_spatial.value = (spatialErr + spatialErrH + spatialErrV) / 3.0f;
float scale = 1.0f;
// Reduce thresholds for HD scenes
if (_imageType > 3)
{
scale = (float)SCALE_TEXTURE_HD;
}
if (_spatial.value > scale * HIGH_TEXTURE)
{
_spatial.level = kHigh;
}
else if (_spatial.value < scale * LOW_TEXTURE)
{
_spatial.level = kLow;
}
else
{
_spatial.level = kDefault;
}
}
void
VCMQmMethod::Coherence()
{
float horizNZ = _contentMetrics->motionHorizontalness;
float distortionNZ = _contentMetrics->motionClusterDistortion;
// Coherence measure: combine horizontalness with cluster distortion
_coherence.value = COH_MAX;
if (distortionNZ > 0.)
{
_coherence.value = horizNZ / distortionNZ;
}
_coherence.value = VCM_MIN(COH_MAX, _coherence.value);
if (_coherence.value < COHERENCE_THR)
{
_coherence.level = kLow;
}
else
{
_coherence.level = kHigh;
}
}
WebRtc_Word8
VCMQmMethod::GetImageType(WebRtc_UWord32 width, WebRtc_UWord32 height)
{
// Match image type
WebRtc_UWord32 imageSize = width * height;
WebRtc_Word8 imageType;
if (imageSize < kFrameSizeTh[0])
{
imageType = 0;
}
else if (imageSize < kFrameSizeTh[1])
{
imageType = 1;
}
else if (imageSize < kFrameSizeTh[2])
{
imageType = 2;
}
else if (imageSize < kFrameSizeTh[3])
{
imageType = 3;
}
else if (imageSize < kFrameSizeTh[4])
{
imageType = 4;
}
else if (imageSize < kFrameSizeTh[5])
{
imageType = 5;
}
else
{
imageType = 6;
}
return imageType;
}
// DONE WITH QM CLASS
//RESOLUTION CLASS
VCMQmResolution::VCMQmResolution()
{
_qm = new VCMResolutionScale();
Reset();
}
VCMQmResolution::~VCMQmResolution()
{
delete _qm;
}
void
VCMQmResolution::ResetRates()
{ {
_sumEncodedBytes = 0; _sumEncodedBytes = 0;
_sumTargetRate = 0.0f; _sumTargetRate = 0.0f;
@ -65,7 +233,7 @@ VCMQmSelect::ResetRates()
} }
void void
VCMQmSelect::Reset() VCMQmResolution::Reset()
{ {
_stateDecFactorSpatial = 1; _stateDecFactorSpatial = 1;
_stateDecFactorTemp = 1; _stateDecFactorTemp = 1;
@ -74,17 +242,14 @@ VCMQmSelect::Reset()
_incomingFrameRate = 0.0f; _incomingFrameRate = 0.0f;
_userFrameRate = 0.0f; _userFrameRate = 0.0f;
_perFrameBandwidth =0.0f; _perFrameBandwidth =0.0f;
_prevTotalRate = 0.0f; ResetRates();
_prevRttTime = 0; ResetQM();
_prevPacketLoss = 0; return;
ResetQM();
ResetRates();
return;
} }
//Initialize after reset of encoder // Initialize rate control quantities after reset of encoder
WebRtc_Word32 WebRtc_Word32
VCMQmSelect::Initialize(float bitRate, float userFrameRate, VCMQmResolution::Initialize(float bitRate, float userFrameRate,
WebRtc_UWord32 width, WebRtc_UWord32 height) WebRtc_UWord32 width, WebRtc_UWord32 height)
{ {
if (userFrameRate == 0.0f || width == 0 || height == 0) if (userFrameRate == 0.0f || width == 0 || height == 0)
@ -98,6 +263,12 @@ VCMQmSelect::Initialize(float bitRate, float userFrameRate,
_width = width; _width = width;
_height = height; _height = height;
// Aspect ratio: used for selection of 1x2,2x1,2x2
_aspectRatio = static_cast<float>(_width) / static_cast<float>(_height);
// Set the imageType for the encoder width/height.
_imageType = GetImageType(_width, _height);
// Initial buffer level // Initial buffer level
_bufferLevel = INIT_BUFFER_LEVEL * _targetBitRate; _bufferLevel = INIT_BUFFER_LEVEL * _targetBitRate;
@ -121,19 +292,9 @@ VCMQmSelect::Initialize(float bitRate, float userFrameRate,
return VCM_OK; return VCM_OK;
} }
WebRtc_Word32 // Update after every encoded frame
VCMQmSelect::SetPreferences(WebRtc_Word8 resolPref)
{
// Preference setting for temporal over spatial resolution
// 100 means temporal, 0 means spatial, 50 is neutral
_userResolutionPref = resolPref;
return VCM_OK;
}
//Update after every encoded frame
void void
VCMQmSelect::UpdateEncodedSize(WebRtc_Word64 encodedSize, VCMQmResolution::UpdateEncodedSize(WebRtc_Word64 encodedSize,
FrameType encodedFrameType) FrameType encodedFrameType)
{ {
// Update encoded size; // Update encoded size;
@ -167,16 +328,16 @@ VCMQmSelect::UpdateEncodedSize(WebRtc_Word64 encodedSize,
*/ */
// Counter for occurrences of low buffer level // Counter for occurrences of low buffer level
if (_bufferLevel <= PERC_BUFFER_THR * INIT_BUFFER_LEVEL * _targetBitRate) if (_bufferLevel <= PERC_BUFFER_THR * OPT_BUFFER_LEVEL * _targetBitRate)
{ {
_lowBufferCnt++; _lowBufferCnt++;
} }
} }
//Update various quantities after SetTargetRates in MediaOpt // Update various quantities after SetTargetRates in MediaOpt
void void
VCMQmSelect::UpdateRates(float targetBitRate, float avgSentBitRate, VCMQmResolution::UpdateRates(float targetBitRate, float avgSentBitRate,
float incomingFrameRate, WebRtc_UWord8 packetLoss) float incomingFrameRate, WebRtc_UWord8 packetLoss)
{ {
@ -219,42 +380,10 @@ VCMQmSelect::UpdateRates(float targetBitRate, float avgSentBitRate,
} }
// Adjust the FEC rate based on the content and the network state
// (packet loss rate, total rate/bandwidth, round trip time).
// Note that packetLoss here is the filtered loss value.
WebRtc_UWord8
VCMQmSelect::AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate,
float frameRate,WebRtc_UWord16 rttTime,
WebRtc_UWord8 packetLoss)
{
// Default: no adjustment
WebRtc_UWord8 codeRateDeltaAdjust = codeRateDelta;
float adjustFec = 1.0f;
// TODO (marpan):
// Set FEC adjustment factor
codeRateDeltaAdjust = static_cast<WebRtc_UWord8>(codeRateDelta * adjustFec);
// Keep track of previous values of network state:
// adjustment may be also based on pattern of changes in network state
_prevTotalRate = totalRate;
_prevRttTime = rttTime;
_prevPacketLoss = packetLoss;
return codeRateDeltaAdjust;
}
void
VCMQmSelect::UpdateContent(const VideoContentMetrics* contentMetrics)
{
_contentMetrics = contentMetrics;
}
// Select the resolution factors: frame size and frame rate change: (QM modes) // Select the resolution factors: frame size and frame rate change: (QM modes)
// Selection is for going back up in resolution, or going down in. // Selection is for going back up in resolution, or going down in.
WebRtc_Word32 WebRtc_Word32
VCMQmSelect::SelectQuality(VCMQualityMode** qm) VCMQmResolution::SelectResolution(VCMResolutionScale** qm)
{ {
if (!_init) if (!_init)
{ {
@ -272,15 +401,11 @@ VCMQmSelect::SelectQuality(VCMQualityMode** qm)
_qm->spatialHeightFact = 1; _qm->spatialHeightFact = 1;
_qm->temporalFact = 1; _qm->temporalFact = 1;
// Update native values // Update native values
_nativeWidth = _contentMetrics->nativeWidth; _nativeWidth = _contentMetrics->nativeWidth;
_nativeHeight = _contentMetrics->nativeHeight; _nativeHeight = _contentMetrics->nativeHeight;
_nativeFrameRate = _contentMetrics->nativeFrameRate; _nativeFrameRate = _contentMetrics->nativeFrameRate;
// Aspect ratio: used for selection of 1x2,2x1,2x2
_aspectRatio = (float)_width / (float)_height;
float avgTargetRate = 0.0f; float avgTargetRate = 0.0f;
float avgIncomingFrameRate = 0.0f; float avgIncomingFrameRate = 0.0f;
float ratioBufferLow = 0.0f; float ratioBufferLow = 0.0f;
@ -317,13 +442,12 @@ VCMQmSelect::SelectQuality(VCMQualityMode** qm)
// for up-sampled spatial dimensions. // for up-sampled spatial dimensions.
// This is needed to get the transRate for going back up in // This is needed to get the transRate for going back up in
// spatial resolution (only 2x2 allowed in this version). // spatial resolution (only 2x2 allowed in this version).
SetMaxRateForQM(2 * _width, 2 * _height); WebRtc_UWord8 imageType2 = GetImageType(2 * _width, 2 * _height);
WebRtc_UWord8 imageType2 = _imageType; WebRtc_UWord32 maxRateQM2 = kMaxRateQm[imageType2];
WebRtc_UWord32 maxRateQM2 = _maxRateQM;
// Set the maximum transitional rate and image type: // Set the maximum transitional rate and image type:
// for the encoder spatial dimensions. // for the encoder spatial dimensions.
SetMaxRateForQM(_width, _height); WebRtc_UWord32 maxRateQM = kMaxRateQm[_imageType];
// Compute class state of the content. // Compute class state of the content.
MotionNFD(); MotionNFD();
@ -342,18 +466,21 @@ VCMQmSelect::SelectQuality(VCMQualityMode** qm)
// Get image class and content class: for going up spatially // Get image class and content class: for going up spatially
WebRtc_UWord8 imageClass2 = 1; WebRtc_UWord8 imageClass2 = 1;
if (imageType2 <= 3) imageClass2 = 0; if (imageType2 <= 3)
{
imageClass2 = 0;
}
WebRtc_UWord8 tableIndex2 = imageClass2 * 9 + contentClass; WebRtc_UWord8 tableIndex2 = imageClass2 * 9 + contentClass;
float scaleTransRate2 = kScaleTransRateQm[tableIndex2]; float scaleTransRate2 = kScaleTransRateQm[tableIndex2];
// Transitonal rate for going down // Transitonal rate for going down
WebRtc_UWord32 estimatedTransRateDown = static_cast<WebRtc_UWord32> WebRtc_UWord32 estimatedTransRateDown = static_cast<WebRtc_UWord32>
(_incomingFrameRate * scaleTransRate * _maxRateQM / 30); (_incomingFrameRate * scaleTransRate * maxRateQM / 30);
// Transitional rate for going up temporally // Transitional rate for going up temporally
WebRtc_UWord32 estimatedTransRateUpT = static_cast<WebRtc_UWord32> WebRtc_UWord32 estimatedTransRateUpT = static_cast<WebRtc_UWord32>
(TRANS_RATE_SCALE_UP_TEMP * 2 * _incomingFrameRate * (TRANS_RATE_SCALE_UP_TEMP * 2 * _incomingFrameRate *
scaleTransRate * _maxRateQM / 30); scaleTransRate * maxRateQM / 30);
// Transitional rate for going up spatially // Transitional rate for going up spatially
WebRtc_UWord32 estimatedTransRateUpS = static_cast<WebRtc_UWord32> WebRtc_UWord32 estimatedTransRateUpS = static_cast<WebRtc_UWord32>
@ -530,7 +657,7 @@ VCMQmSelect::SelectQuality(VCMQualityMode** qm)
} }
WebRtc_Word32 WebRtc_Word32
VCMQmSelect::SelectSpatialDirectionMode(float transRate) VCMQmResolution::SelectSpatialDirectionMode(float transRate)
{ {
// Default is 1x2 (H) // Default is 1x2 (H)
@ -579,156 +706,82 @@ VCMQmSelect::SelectSpatialDirectionMode(float transRate)
return VCM_OK; return VCM_OK;
} }
void // DONE WITH RESOLUTION CLASS
VCMQmSelect::Coherence()
// ROBUSTNESS CLASS
VCMQmRobustness::VCMQmRobustness()
{ {
float horizNZ = _contentMetrics->motionHorizontalness; Reset();
float distortionNZ = _contentMetrics->motionClusterDistortion; }
// Coherence measure: combine horizontalness with cluster distortion VCMQmRobustness::~VCMQmRobustness()
_coherence.value = COH_MAX; {
if (distortionNZ > 0.)
{
_coherence.value = horizNZ / distortionNZ;
}
_coherence.value = VCM_MIN(COH_MAX, _coherence.value);
if (_coherence.value < COHERENCE_THR)
{
_coherence.level = kLow;
}
else
{
_coherence.level = kHigh;
}
} }
void void
VCMQmSelect::MotionNFD() VCMQmRobustness::Reset()
{ {
_motion.value = _contentMetrics->motionMagnitudeNZ; _prevTotalRate = 0.0f;
_prevRttTime = 0;
// Determine motion level _prevPacketLoss = 0;
if (_motion.value < LOW_MOTION_NFD) ResetQM();
{ return;
_motion.level = kLow;
}
else if (_motion.value > HIGH_MOTION_NFD)
{
_motion.level = kHigh;
}
else
{
_motion.level = kDefault;
}
} }
void // Adjust the FEC rate based on the content and the network state
VCMQmSelect::Motion() // (packet loss rate, total rate/bandwidth, round trip time).
// Note that packetLoss here is the filtered loss value.
WebRtc_UWord8
VCMQmRobustness::AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate,
float frameRate,WebRtc_UWord32 rttTime,
WebRtc_UWord8 packetLoss)
{ {
if (_contentMetrics == NULL)
{
return VCM_OK;
}
float sizeZeroMotion = _contentMetrics->sizeZeroMotion; // Default: no adjustment
float motionMagNZ = _contentMetrics->motionMagnitudeNZ; WebRtc_UWord8 codeRateDeltaAdjust = codeRateDelta;
float adjustFec = 1.0f;
// Take product of size and magnitude with equal weight // Compute class state of the content.
_motion.value = (1.0f - sizeZeroMotion) * motionMagNZ; MotionNFD();
Spatial();
// Stabilize: motionMagNZ could be large when only a // TODO (marpan):
// few motion blocks are non-zero // Set FEC adjustment factor
_stationaryMotion = false;
if (sizeZeroMotion > HIGH_ZERO_MOTION_SIZE) codeRateDeltaAdjust = static_cast<WebRtc_UWord8>(codeRateDelta * adjustFec);
{
_motion.value = 0.0f; // Keep track of previous values of network state:
_stationaryMotion = true; // adjustment may be also based on pattern of changes in network state
} _prevTotalRate = totalRate;
// Determine motion level _prevRttTime = rttTime;
if (_motion.value < LOW_MOTION) _prevPacketLoss = packetLoss;
{
_motion.level = kLow; _prevCodeRateDelta = codeRateDelta;
}
else if (_motion.value > HIGH_MOTION) return codeRateDeltaAdjust;
{
_motion.level = kHigh;
}
else
{
_motion.level = kDefault;
}
} }
// Set the UEP (unequal-protection) on/off for the FEC
void bool
VCMQmSelect::Spatial() VCMQmRobustness::SetUepProtection(WebRtc_UWord8 codeRateDelta, float totalRate,
WebRtc_UWord8 packetLoss, bool frameType)
{ {
float spatialErr = _contentMetrics->spatialPredErr; if (_contentMetrics == NULL)
float spatialErrH = _contentMetrics->spatialPredErrH;
float spatialErrV = _contentMetrics->spatialPredErrV;
// Spatial measure: take average of 3 prediction errors
_spatial.value = (spatialErr + spatialErrH + spatialErrV) / 3.0f;
float scale = 1.0f;
// Reduce thresholds for HD scenes
if (_imageType > 3)
{ {
scale = (float)SCALE_TEXTURE_HD; return VCM_OK;
} }
if (_spatial.value > scale * HIGH_TEXTURE) // Default: UEP on
{ bool uepProtection = true;
_spatial.level = kHigh;
}
else if (_spatial.value < scale * LOW_TEXTURE)
{
_spatial.level = kLow;
}
else
{
_spatial.level = kDefault;
}
}
return uepProtection;
WebRtc_Word32
VCMQmSelect::SetMaxRateForQM(WebRtc_UWord32 width, WebRtc_UWord32 height)
{
// Match image type
WebRtc_UWord32 imageSize = width * height;
if (imageSize < kFrameSizeTh[0])
{
_imageType = 0;
}
else if (imageSize < kFrameSizeTh[1])
{
_imageType = 1;
}
else if (imageSize < kFrameSizeTh[2])
{
_imageType = 2;
}
else if (imageSize < kFrameSizeTh[3])
{
_imageType = 3;
}
else if (imageSize < kFrameSizeTh[4])
{
_imageType = 4;
}
else if (imageSize < kFrameSizeTh[5])
{
_imageType = 5;
}
else
{
_imageType = 6;
}
// Set max rate based on image size
_maxRateQM = kMaxRateQm[_imageType];
return VCM_OK;
} }
} // end of namespace } // end of namespace

183
src/modules/video_coding/main/source/qm_select.h
View File

@ -13,25 +13,19 @@
#include "typedefs.h" #include "typedefs.h"
#include "common_types.h" #include "common_types.h"
/************************/ /******************************************************/
/* Quality Modes */ /* Quality Modes: Resolution and Robustness settings */
/**********************/ /******************************************************/
namespace webrtc namespace webrtc
{ {
struct VideoContentMetrics; struct VideoContentMetrics;
struct VCMQualityMode struct VCMResolutionScale
{ {
VCMQualityMode():spatialWidthFact(1), spatialHeightFact(1), VCMResolutionScale(): spatialWidthFact(1), spatialHeightFact(1),
temporalFact(1){} temporalFact(1){}
void Reset()
{
spatialWidthFact = 1;
spatialHeightFact = 1;
temporalFact = 1;
}
WebRtc_UWord16 spatialWidthFact; WebRtc_UWord16 spatialWidthFact;
WebRtc_UWord16 spatialHeightFact; WebRtc_UWord16 spatialHeightFact;
@ -59,53 +53,20 @@ struct VCMContFeature
VCMMagValues level; VCMMagValues level;
}; };
class VCMQmSelect // QmMethod class: main class for resolution and robustness settings
class VCMQmMethod
{ {
public: public:
VCMQmSelect(); VCMQmMethod();
~VCMQmSelect(); ~VCMQmMethod();
// Initialize: // Reset values
WebRtc_Word32 Initialize(float bitRate, float userFrameRate,
WebRtc_UWord32 width, WebRtc_UWord32 height);
// Allow the user to set preferences: favor frame rate/resolution
WebRtc_Word32 SetPreferences(WebRtc_Word8 resolPref);
// Extract ST (spatio-temporal) QM behavior and make decision
// Inputs: qm: Reference to the quality modes pointer
WebRtc_Word32 SelectQuality(VCMQualityMode** qm);
// Update QM with actual bit rate
// (size of the latest encoded frame) and frame type.
void UpdateEncodedSize(WebRtc_Word64 encodedSize,
FrameType encodedFrameType);
// Update QM with new bit/frame/loss rates from SetTargetRates
void UpdateRates(float targetBitRate, float avgSentRate,
float incomingFrameRate, WebRtc_UWord8 packetLoss);
// Update QM with the content metrics
void UpdateContent(const VideoContentMetrics* contentMetrics);
// Adjust FEC rate based on content
WebRtc_UWord8 AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate,
float frameRate, WebRtc_UWord16 rttTime,
WebRtc_UWord8 packetLoss);
// Select 1x2,2x2,2x2 spatial sampling mode
WebRtc_Word32 SelectSpatialDirectionMode(float transRate);
// Reset values prior to QMSelect
void ResetQM(); void ResetQM();
virtual void Reset() = 0;
// Reset rate quantities and counter values after every QMSelect call // Update with the content metrics
void ResetRates(); void UpdateContent(const VideoContentMetrics* contentMetrics);
// Reset all
void Reset();
private:
// Compute spatial texture magnitude and level // Compute spatial texture magnitude and level
void Spatial(); void Spatial();
@ -119,36 +80,84 @@ private:
// Compute coherence magnitude and level // Compute coherence magnitude and level
void Coherence(); void Coherence();
// Set the max rate for QM selection // Get the imageType (CIF, VGA, HD, etc) for the system width/height
WebRtc_Word32 SetMaxRateForQM(WebRtc_UWord32 width, WebRtc_UWord32 height); WebRtc_Word8 GetImageType(WebRtc_UWord32 width, WebRtc_UWord32 height);
// Content Data // Content Data
const VideoContentMetrics* _contentMetrics; const VideoContentMetrics* _contentMetrics;
// Encoder rate control parameters, network parameters // Encoder and native frame sizes, frame rate, aspect ratio, imageType
WebRtc_UWord32 _width;
WebRtc_UWord32 _height;
WebRtc_UWord32 _nativeWidth;
WebRtc_UWord32 _nativeHeight;
WebRtc_UWord32 _nativeFrameRate;
float _aspectRatio;
// Image type for the current encoder system size.
WebRtc_UWord8 _imageType;
// Content L/M/H values. stationary flag
VCMContFeature _motion;
VCMContFeature _spatial;
VCMContFeature _coherence;
bool _stationaryMotion;
bool _init;
};
// Resolution settings class
class VCMQmResolution : public VCMQmMethod
{
public:
VCMQmResolution();
~VCMQmResolution();
// Reset all quantities
virtual void Reset();
// Reset rate quantities and counter values after every Select Quality call
void ResetRates();
// Initialize rate control quantities after re-init of encoder.
WebRtc_Word32 Initialize(float bitRate, float userFrameRate,
WebRtc_UWord32 width, WebRtc_UWord32 height);
// Update QM with actual bit rate (size of the latest encoded frame)
// and frame type, after every encoded frame.
void UpdateEncodedSize(WebRtc_Word64 encodedSize,
FrameType encodedFrameType);
// Update QM with new bit/frame/loss rates every ~1 sec from SetTargetRates
void UpdateRates(float targetBitRate, float avgSentRate,
float incomingFrameRate, WebRtc_UWord8 packetLoss);
// Extract ST (spatio-temporal) QM behavior and make decision
// Inputs: qm: Reference to the quality modes pointer
// Output: the spatial and/or temporal scale change
WebRtc_Word32 SelectResolution(VCMResolutionScale** qm);
// Select 1x2,2x2,2x2 spatial sampling mode
WebRtc_Word32 SelectSpatialDirectionMode(float transRate);
private:
// Encoder rate control parameter
float _targetBitRate; float _targetBitRate;
float _userFrameRate; float _userFrameRate;
float _incomingFrameRate; float _incomingFrameRate;
float _perFrameBandwidth; float _perFrameBandwidth;
float _bufferLevel; float _bufferLevel;
// Data accumulated every ~1sec from MediaOpt
float _sumTargetRate; float _sumTargetRate;
float _sumIncomingFrameRate; float _sumIncomingFrameRate;
float _sumSeqRateMM; float _sumSeqRateMM;
float _sumFrameRateMM; float _sumFrameRateMM;
float _sumPacketLoss; float _sumPacketLoss;
float _prevTotalRate;
WebRtc_UWord16 _prevRttTime;
WebRtc_UWord8 _prevPacketLoss;
WebRtc_Word64 _sumEncodedBytes; WebRtc_Word64 _sumEncodedBytes;
// Encoder and native frame sizes // Resolution state parameters
WebRtc_UWord32 _width;
WebRtc_UWord32 _height;
WebRtc_UWord32 _nativeWidth;
WebRtc_UWord32 _nativeHeight;
WebRtc_UWord8 _stateDecFactorSpatial; WebRtc_UWord8 _stateDecFactorSpatial;
WebRtc_UWord32 _nativeFrameRate;
WebRtc_UWord8 _stateDecFactorTemp; WebRtc_UWord8 _stateDecFactorTemp;
// Counters // Counters
@ -157,24 +166,36 @@ private:
WebRtc_UWord32 _updateRateCnt; WebRtc_UWord32 _updateRateCnt;
WebRtc_UWord32 _lowBufferCnt; WebRtc_UWord32 _lowBufferCnt;
// Content L/M/H values VCMResolutionScale* _qm;
VCMContFeature _motion; };
VCMContFeature _spatial;
VCMContFeature _coherence;
bool _stationaryMotion;
// Aspect ratio // Robustness settings class
float _aspectRatio;
// Max rate to saturate the transitionalRate class VCMQmRobustness : public VCMQmMethod
WebRtc_UWord32 _maxRateQM; {
WebRtc_UWord8 _imageType; public:
VCMQmRobustness();
~VCMQmRobustness();
// User preference for resolution or qmax change virtual void Reset();
WebRtc_UWord8 _userResolutionPref;
bool _init;
VCMQualityMode* _qm;
// Adjust FEC rate based on content: every ~1 sec from SetTargetRates
WebRtc_UWord8 AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate,
float frameRate, WebRtc_UWord32 rttTime,
WebRtc_UWord8 packetLoss);
// Set the UEP protection on/off
bool SetUepProtection(WebRtc_UWord8 codeRateDelta, float totalRate,
WebRtc_UWord8 packetLoss, bool frameType);
private:
// Previous state of network parameters
float _prevTotalRate;
WebRtc_UWord32 _prevRttTime;
WebRtc_UWord8 _prevPacketLoss;
// Previous FEC rate
WebRtc_UWord8 _prevCodeRateDelta;
}; };
} // namespace webrtc } // namespace webrtc

3
src/modules/video_coding/main/source/qm_select_data.h
View File

@ -28,6 +28,9 @@ namespace webrtc
// Initial level of buffer in secs: should corresponds to wrapper settings // Initial level of buffer in secs: should corresponds to wrapper settings
#define INIT_BUFFER_LEVEL 0.5 #define INIT_BUFFER_LEVEL 0.5
// Optimal level of buffer in secs: should corresponds to wrapper settings
#define OPT_BUFFER_LEVEL 0.6
// Threshold of (max) buffer size below which we consider too low (underflow) // Threshold of (max) buffer size below which we consider too low (underflow)
#define PERC_BUFFER_THR 0.10 #define PERC_BUFFER_THR 0.10