Modified media_opt and qm_select to allow for robustness settings
based on the content metrics. Re-organized the class structure in qm_select into a resolution class and robustness settings class, both derived from a main (qm/content analysis) class. Review URL: http://webrtc-codereview.appspot.com/55006 git-svn-id: http://webrtc.googlecode.com/svn/trunk@188 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
marpan@google.com
parent
6bed064e2f
commit
86548c62e9
@ -21,6 +21,13 @@
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
void
|
||||
VCMProtectionMethod::UpdateContentMetrics(
|
||||
const VideoContentMetrics* contentMetrics)
|
||||
{
|
||||
_qmRobustness->UpdateContent(contentMetrics);
|
||||
}
|
||||
|
||||
bool
|
||||
VCMProtectionMethod::BetterThan(VCMProtectionMethod *pm)
|
||||
{
|
||||
@ -417,6 +424,10 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters)
|
||||
codeRateDelta = plossMax - 1;
|
||||
}
|
||||
|
||||
codeRateDelta = _qmRobustness->AdjustFecFactor(codeRateDelta, bitRate,
|
||||
parameters->frameRate,
|
||||
parameters->rtt, packetLoss);
|
||||
|
||||
// For Key frame:
|
||||
// Effectively at a higher rate, so we scale/boost the rate
|
||||
// The boost factor may depend on several factors: ratio of packet
|
||||
@ -462,6 +473,14 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters)
|
||||
_protectionFactorK = codeRateKey;
|
||||
_protectionFactorD = codeRateDelta;
|
||||
|
||||
|
||||
// Set the UEP protection on/off for Key and Delta frames
|
||||
_uepKey = _qmRobustness->SetUepProtection(codeRateKey, bitRate,
|
||||
packetLoss, 0);
|
||||
|
||||
_uepDelta = _qmRobustness->SetUepProtection(codeRateKey, bitRate,
|
||||
packetLoss, 1);
|
||||
|
||||
// DONE WITH FEC PROTECTION SETTINGS
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -17,6 +17,7 @@
|
||||
#include "exp_filter.h"
|
||||
#include "internal_defines.h"
|
||||
#include "tick_time.h"
|
||||
#include "qm_select.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdlib>
|
||||
@ -97,8 +98,10 @@ public:
|
||||
//friend VCMProtectionMethod;
|
||||
VCMProtectionMethod(VCMProtectionMethodEnum type) : _protectionFactorK(0),
|
||||
_protectionFactorD(0), _residualPacketLoss(0.0), _scaleProtKey(2.0),
|
||||
_maxPayloadSize(1460), _efficiency(0), _score(0), _type(type) {}
|
||||
virtual ~VCMProtectionMethod() {}
|
||||
_maxPayloadSize(1460), _efficiency(0), _score(0), _type(type),
|
||||
_uepKey(0), _uepDelta(1)
|
||||
{_qmRobustness = new VCMQmRobustness();}
|
||||
virtual ~VCMProtectionMethod() { delete _qmRobustness;}
|
||||
|
||||
// Updates the efficiency of the method using the parameters provided
|
||||
//
|
||||
@ -142,6 +145,9 @@ public:
|
||||
// Return value : Required protectionFactor for delta frame
|
||||
virtual WebRtc_UWord8 RequiredProtectionFactorD() { return _protectionFactorD; }
|
||||
|
||||
// Updates content metrics
|
||||
void UpdateContentMetrics(const VideoContentMetrics* contentMetrics);
|
||||
|
||||
WebRtc_UWord8 _effectivePacketLoss;
|
||||
WebRtc_UWord8 _protectionFactorK;
|
||||
WebRtc_UWord8 _protectionFactorD;
|
||||
@ -149,6 +155,10 @@ public:
|
||||
float _scaleProtKey;
|
||||
WebRtc_Word32 _maxPayloadSize;
|
||||
|
||||
VCMQmRobustness* _qmRobustness;
|
||||
bool _uepKey;
|
||||
bool _uepDelta;
|
||||
|
||||
protected:
|
||||
float _efficiency;
|
||||
float _score;
|
||||
|
||||
@ -46,7 +46,7 @@ _lastChangeTime(0)
|
||||
_frameDropper = new VCMFrameDropper(_id);
|
||||
_lossProtLogic = new VCMLossProtectionLogic();
|
||||
_content = new VCMContentMetricsProcessing();
|
||||
_qms = new VCMQmSelect();
|
||||
_qmResolution = new VCMQmResolution();
|
||||
}
|
||||
|
||||
VCMMediaOptimization::~VCMMediaOptimization(void)
|
||||
@ -55,7 +55,7 @@ VCMMediaOptimization::~VCMMediaOptimization(void)
|
||||
delete _lossProtLogic;
|
||||
delete _frameDropper;
|
||||
delete _content;
|
||||
delete _qms;
|
||||
delete _qmResolution;
|
||||
}
|
||||
|
||||
WebRtc_Word32
|
||||
@ -67,7 +67,7 @@ VCMMediaOptimization::Reset()
|
||||
_lossProtLogic->Reset();
|
||||
_frameDropper->SetRates(0, 0);
|
||||
_content->Reset();
|
||||
_qms->Reset();
|
||||
_qmResolution->Reset();
|
||||
_lossProtLogic->UpdateFrameRate(_incomingFrameRate);
|
||||
_lossProtLogic->Reset();
|
||||
_sendStatisticsZeroEncode = 0;
|
||||
@ -135,6 +135,10 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
|
||||
selectedMethod->Type() == kNackFec ))
|
||||
{
|
||||
|
||||
// Update protection method with content metrics
|
||||
selectedMethod->UpdateContentMetrics(_content->ShortTermAvgData());
|
||||
|
||||
|
||||
// Update method will compute the robustness settings for the given
|
||||
// protection method and the overhead cost
|
||||
// the protection method is set by the user via SetVideoProtection.
|
||||
@ -197,8 +201,8 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate,
|
||||
if (_enableQm)
|
||||
{
|
||||
//Update QM with rates
|
||||
_qms->UpdateRates((float)_targetBitRate, _avgSentBitRateBps,
|
||||
_incomingFrameRate, _fractionLost);
|
||||
_qmResolution->UpdateRates((float)_targetBitRate, _avgSentBitRateBps,
|
||||
_incomingFrameRate, _fractionLost);
|
||||
//Check for QM selection
|
||||
bool selectQM = checkStatusForQMchange();
|
||||
if (selectQM)
|
||||
@ -249,11 +253,12 @@ VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, WebRtc_Word3
|
||||
_lossProtLogic->UpdateFrameSize(width, height);
|
||||
_frameDropper->Reset();
|
||||
_frameDropper->SetRates(static_cast<float>(bitRate), static_cast<float>(frameRate));
|
||||
_userFrameRate = (float)frameRate;
|
||||
_userFrameRate = static_cast<float>(frameRate);
|
||||
_codecWidth = width;
|
||||
_codecHeight = height;
|
||||
WebRtc_Word32 ret = VCM_OK;
|
||||
ret = _qms->Initialize((float)_targetBitRate, _userFrameRate, _codecWidth, _codecHeight);
|
||||
ret = _qmResolution->Initialize((float)_targetBitRate, _userFrameRate,
|
||||
_codecWidth, _codecHeight);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -420,7 +425,8 @@ VCMMediaOptimization::UpdateWithEncodedData(WebRtc_Word32 encodedLength,
|
||||
if (_enableQm)
|
||||
{
|
||||
// update quality select with encoded length
|
||||
_qms->UpdateEncodedSize(encodedLength, encodedFrameType);
|
||||
_qmResolution->UpdateEncodedSize(encodedLength,
|
||||
encodedFrameType);
|
||||
}
|
||||
}
|
||||
if (!deltaFrame && encodedLength > 0)
|
||||
@ -525,7 +531,7 @@ VCMMediaOptimization::updateContentData(const VideoContentMetrics *contentMetric
|
||||
{
|
||||
//No QM if metrics are NULL
|
||||
_enableQm = false;
|
||||
_qms->Reset();
|
||||
_qmResolution->Reset();
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -537,14 +543,14 @@ WebRtc_Word32
|
||||
VCMMediaOptimization::SelectQuality()
|
||||
{
|
||||
// Reset quantities for QM select
|
||||
_qms->ResetQM();
|
||||
_qmResolution->ResetQM();
|
||||
|
||||
// Update QM will long-term averaged content metrics.
|
||||
_qms->UpdateContent(_content->LongTermAvgData());
|
||||
_qmResolution->UpdateContent(_content->LongTermAvgData());
|
||||
|
||||
// Select quality mode
|
||||
VCMQualityMode* qm = NULL;
|
||||
WebRtc_Word32 ret = _qms->SelectQuality(&qm);
|
||||
VCMResolutionScale* qm = NULL;
|
||||
WebRtc_Word32 ret = _qmResolution->SelectResolution(&qm);
|
||||
if (ret < 0)
|
||||
{
|
||||
return ret;
|
||||
@ -554,7 +560,7 @@ VCMMediaOptimization::SelectQuality()
|
||||
QMUpdate(qm);
|
||||
|
||||
// Reset all the rate and related frame counters quantities
|
||||
_qms->ResetRates();
|
||||
_qmResolution->ResetRates();
|
||||
|
||||
// Reset counters
|
||||
_lastQMUpdateTime = VCMTickTime::MillisecondTimestamp();
|
||||
@ -592,7 +598,7 @@ VCMMediaOptimization::checkStatusForQMchange()
|
||||
}
|
||||
|
||||
bool
|
||||
VCMMediaOptimization::QMUpdate(VCMQualityMode* qm)
|
||||
VCMMediaOptimization::QMUpdate(VCMResolutionScale* qm)
|
||||
{
|
||||
// Check for no change
|
||||
if (qm->spatialHeightFact == 1 &&
|
||||
@ -606,7 +612,9 @@ VCMMediaOptimization::QMUpdate(VCMQualityMode* qm)
|
||||
VideoContentMetrics* cm = _content->LongTermAvgData();
|
||||
|
||||
// Temporal
|
||||
WebRtc_UWord32 frameRate = static_cast<WebRtc_UWord32>(_incomingFrameRate + 0.5f);
|
||||
WebRtc_UWord32 frameRate = static_cast<WebRtc_UWord32>
|
||||
(_incomingFrameRate + 0.5f);
|
||||
|
||||
// Check if go back up in temporal resolution
|
||||
if (qm->temporalFact == 0)
|
||||
{
|
||||
|
||||
@ -159,7 +159,7 @@ private:
|
||||
* verify if QM settings differ from default, i.e. if an update is required
|
||||
* Compute actual values, as will be sent to the encoder
|
||||
*/
|
||||
bool QMUpdate(VCMQualityMode* qm);
|
||||
bool QMUpdate(VCMResolutionScale* qm);
|
||||
/**
|
||||
* check if we should make a QM change
|
||||
* will return 1 if yes, 0 otherwise
|
||||
@ -207,7 +207,7 @@ private:
|
||||
WebRtc_UWord32 _deltaFrameCnt;
|
||||
|
||||
VCMContentMetricsProcessing* _content;
|
||||
VCMQmSelect* _qms;
|
||||
VCMQmResolution* _qmResolution;
|
||||
|
||||
WebRtc_Word64 _lastQMUpdateTime;
|
||||
WebRtc_Word64 _lastChangeTime; // content or user triggered
|
||||
|
||||
@ -20,36 +20,204 @@
|
||||
|
||||
namespace webrtc {
|
||||
|
||||
VCMQmSelect::VCMQmSelect()
|
||||
// QM-METHOD class
|
||||
|
||||
VCMQmMethod::VCMQmMethod()
|
||||
{
|
||||
_qm = new VCMQualityMode();
|
||||
_contentMetrics = new VideoContentMetrics();
|
||||
Reset();
|
||||
ResetQM();
|
||||
}
|
||||
|
||||
VCMQmSelect::~VCMQmSelect()
|
||||
VCMQmMethod::~VCMQmMethod()
|
||||
{
|
||||
delete _qm;
|
||||
delete _contentMetrics;
|
||||
}
|
||||
|
||||
void
|
||||
VCMQmSelect::ResetQM()
|
||||
VCMQmMethod::ResetQM()
|
||||
{
|
||||
_motion.Reset();
|
||||
_spatial.Reset();
|
||||
_coherence.Reset();
|
||||
_stationaryMotion = 0;
|
||||
_aspectRatio = 1;
|
||||
_maxRateQM = 0;
|
||||
_imageType = 1;
|
||||
_userResolutionPref = 50; // Neutral
|
||||
_qm->Reset();
|
||||
_imageType = 2;
|
||||
return;
|
||||
}
|
||||
|
||||
void
|
||||
VCMQmSelect::ResetRates()
|
||||
VCMQmMethod::UpdateContent(const VideoContentMetrics* contentMetrics)
|
||||
{
|
||||
_contentMetrics = contentMetrics;
|
||||
}
|
||||
|
||||
void
|
||||
VCMQmMethod::MotionNFD()
|
||||
{
|
||||
_motion.value = _contentMetrics->motionMagnitudeNZ;
|
||||
|
||||
// Determine motion level
|
||||
if (_motion.value < LOW_MOTION_NFD)
|
||||
{
|
||||
_motion.level = kLow;
|
||||
}
|
||||
else if (_motion.value > HIGH_MOTION_NFD)
|
||||
{
|
||||
_motion.level = kHigh;
|
||||
}
|
||||
else
|
||||
{
|
||||
_motion.level = kDefault;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void
|
||||
VCMQmMethod::Motion()
|
||||
{
|
||||
|
||||
float sizeZeroMotion = _contentMetrics->sizeZeroMotion;
|
||||
float motionMagNZ = _contentMetrics->motionMagnitudeNZ;
|
||||
|
||||
// Take product of size and magnitude with equal weight
|
||||
_motion.value = (1.0f - sizeZeroMotion) * motionMagNZ;
|
||||
|
||||
// Stabilize: motionMagNZ could be large when only a
|
||||
// few motion blocks are non-zero
|
||||
_stationaryMotion = false;
|
||||
if (sizeZeroMotion > HIGH_ZERO_MOTION_SIZE)
|
||||
{
|
||||
_motion.value = 0.0f;
|
||||
_stationaryMotion = true;
|
||||
}
|
||||
// Determine motion level
|
||||
if (_motion.value < LOW_MOTION)
|
||||
{
|
||||
_motion.level = kLow;
|
||||
}
|
||||
else if (_motion.value > HIGH_MOTION)
|
||||
{
|
||||
_motion.level = kHigh;
|
||||
}
|
||||
else
|
||||
{
|
||||
_motion.level = kDefault;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
VCMQmMethod::Spatial()
|
||||
{
|
||||
float spatialErr = _contentMetrics->spatialPredErr;
|
||||
float spatialErrH = _contentMetrics->spatialPredErrH;
|
||||
float spatialErrV = _contentMetrics->spatialPredErrV;
|
||||
// Spatial measure: take average of 3 prediction errors
|
||||
_spatial.value = (spatialErr + spatialErrH + spatialErrV) / 3.0f;
|
||||
|
||||
float scale = 1.0f;
|
||||
// Reduce thresholds for HD scenes
|
||||
if (_imageType > 3)
|
||||
{
|
||||
scale = (float)SCALE_TEXTURE_HD;
|
||||
}
|
||||
|
||||
if (_spatial.value > scale * HIGH_TEXTURE)
|
||||
{
|
||||
_spatial.level = kHigh;
|
||||
}
|
||||
else if (_spatial.value < scale * LOW_TEXTURE)
|
||||
{
|
||||
_spatial.level = kLow;
|
||||
}
|
||||
else
|
||||
{
|
||||
_spatial.level = kDefault;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
VCMQmMethod::Coherence()
|
||||
{
|
||||
float horizNZ = _contentMetrics->motionHorizontalness;
|
||||
float distortionNZ = _contentMetrics->motionClusterDistortion;
|
||||
|
||||
// Coherence measure: combine horizontalness with cluster distortion
|
||||
_coherence.value = COH_MAX;
|
||||
if (distortionNZ > 0.)
|
||||
{
|
||||
_coherence.value = horizNZ / distortionNZ;
|
||||
}
|
||||
_coherence.value = VCM_MIN(COH_MAX, _coherence.value);
|
||||
|
||||
if (_coherence.value < COHERENCE_THR)
|
||||
{
|
||||
_coherence.level = kLow;
|
||||
}
|
||||
else
|
||||
{
|
||||
_coherence.level = kHigh;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
WebRtc_Word8
|
||||
VCMQmMethod::GetImageType(WebRtc_UWord32 width, WebRtc_UWord32 height)
|
||||
{
|
||||
// Match image type
|
||||
WebRtc_UWord32 imageSize = width * height;
|
||||
WebRtc_Word8 imageType;
|
||||
|
||||
if (imageSize < kFrameSizeTh[0])
|
||||
{
|
||||
imageType = 0;
|
||||
}
|
||||
else if (imageSize < kFrameSizeTh[1])
|
||||
{
|
||||
imageType = 1;
|
||||
}
|
||||
else if (imageSize < kFrameSizeTh[2])
|
||||
{
|
||||
imageType = 2;
|
||||
}
|
||||
else if (imageSize < kFrameSizeTh[3])
|
||||
{
|
||||
imageType = 3;
|
||||
}
|
||||
else if (imageSize < kFrameSizeTh[4])
|
||||
{
|
||||
imageType = 4;
|
||||
}
|
||||
else if (imageSize < kFrameSizeTh[5])
|
||||
{
|
||||
imageType = 5;
|
||||
}
|
||||
else
|
||||
{
|
||||
imageType = 6;
|
||||
}
|
||||
|
||||
return imageType;
|
||||
}
|
||||
|
||||
// DONE WITH QM CLASS
|
||||
|
||||
|
||||
//RESOLUTION CLASS
|
||||
|
||||
VCMQmResolution::VCMQmResolution()
|
||||
{
|
||||
_qm = new VCMResolutionScale();
|
||||
Reset();
|
||||
}
|
||||
|
||||
VCMQmResolution::~VCMQmResolution()
|
||||
{
|
||||
delete _qm;
|
||||
}
|
||||
|
||||
void
|
||||
VCMQmResolution::ResetRates()
|
||||
{
|
||||
_sumEncodedBytes = 0;
|
||||
_sumTargetRate = 0.0f;
|
||||
@ -65,7 +233,7 @@ VCMQmSelect::ResetRates()
|
||||
}
|
||||
|
||||
void
|
||||
VCMQmSelect::Reset()
|
||||
VCMQmResolution::Reset()
|
||||
{
|
||||
_stateDecFactorSpatial = 1;
|
||||
_stateDecFactorTemp = 1;
|
||||
@ -74,17 +242,14 @@ VCMQmSelect::Reset()
|
||||
_incomingFrameRate = 0.0f;
|
||||
_userFrameRate = 0.0f;
|
||||
_perFrameBandwidth =0.0f;
|
||||
_prevTotalRate = 0.0f;
|
||||
_prevRttTime = 0;
|
||||
_prevPacketLoss = 0;
|
||||
ResetQM();
|
||||
ResetRates();
|
||||
return;
|
||||
ResetRates();
|
||||
ResetQM();
|
||||
return;
|
||||
}
|
||||
|
||||
//Initialize after reset of encoder
|
||||
// Initialize rate control quantities after reset of encoder
|
||||
WebRtc_Word32
|
||||
VCMQmSelect::Initialize(float bitRate, float userFrameRate,
|
||||
VCMQmResolution::Initialize(float bitRate, float userFrameRate,
|
||||
WebRtc_UWord32 width, WebRtc_UWord32 height)
|
||||
{
|
||||
if (userFrameRate == 0.0f || width == 0 || height == 0)
|
||||
@ -98,6 +263,12 @@ VCMQmSelect::Initialize(float bitRate, float userFrameRate,
|
||||
_width = width;
|
||||
_height = height;
|
||||
|
||||
// Aspect ratio: used for selection of 1x2,2x1,2x2
|
||||
_aspectRatio = static_cast<float>(_width) / static_cast<float>(_height);
|
||||
|
||||
// Set the imageType for the encoder width/height.
|
||||
_imageType = GetImageType(_width, _height);
|
||||
|
||||
// Initial buffer level
|
||||
_bufferLevel = INIT_BUFFER_LEVEL * _targetBitRate;
|
||||
|
||||
@ -121,19 +292,9 @@ VCMQmSelect::Initialize(float bitRate, float userFrameRate,
|
||||
return VCM_OK;
|
||||
}
|
||||
|
||||
WebRtc_Word32
|
||||
VCMQmSelect::SetPreferences(WebRtc_Word8 resolPref)
|
||||
{
|
||||
// Preference setting for temporal over spatial resolution
|
||||
// 100 means temporal, 0 means spatial, 50 is neutral
|
||||
_userResolutionPref = resolPref;
|
||||
|
||||
return VCM_OK;
|
||||
}
|
||||
|
||||
//Update after every encoded frame
|
||||
// Update after every encoded frame
|
||||
void
|
||||
VCMQmSelect::UpdateEncodedSize(WebRtc_Word64 encodedSize,
|
||||
VCMQmResolution::UpdateEncodedSize(WebRtc_Word64 encodedSize,
|
||||
FrameType encodedFrameType)
|
||||
{
|
||||
// Update encoded size;
|
||||
@ -167,16 +328,16 @@ VCMQmSelect::UpdateEncodedSize(WebRtc_Word64 encodedSize,
|
||||
*/
|
||||
|
||||
// Counter for occurrences of low buffer level
|
||||
if (_bufferLevel <= PERC_BUFFER_THR * INIT_BUFFER_LEVEL * _targetBitRate)
|
||||
if (_bufferLevel <= PERC_BUFFER_THR * OPT_BUFFER_LEVEL * _targetBitRate)
|
||||
{
|
||||
_lowBufferCnt++;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//Update various quantities after SetTargetRates in MediaOpt
|
||||
// Update various quantities after SetTargetRates in MediaOpt
|
||||
void
|
||||
VCMQmSelect::UpdateRates(float targetBitRate, float avgSentBitRate,
|
||||
VCMQmResolution::UpdateRates(float targetBitRate, float avgSentBitRate,
|
||||
float incomingFrameRate, WebRtc_UWord8 packetLoss)
|
||||
{
|
||||
|
||||
@ -219,42 +380,10 @@ VCMQmSelect::UpdateRates(float targetBitRate, float avgSentBitRate,
|
||||
|
||||
}
|
||||
|
||||
// Adjust the FEC rate based on the content and the network state
|
||||
// (packet loss rate, total rate/bandwidth, round trip time).
|
||||
// Note that packetLoss here is the filtered loss value.
|
||||
WebRtc_UWord8
|
||||
VCMQmSelect::AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate,
|
||||
float frameRate,WebRtc_UWord16 rttTime,
|
||||
WebRtc_UWord8 packetLoss)
|
||||
{
|
||||
// Default: no adjustment
|
||||
WebRtc_UWord8 codeRateDeltaAdjust = codeRateDelta;
|
||||
float adjustFec = 1.0f;
|
||||
|
||||
// TODO (marpan):
|
||||
// Set FEC adjustment factor
|
||||
|
||||
codeRateDeltaAdjust = static_cast<WebRtc_UWord8>(codeRateDelta * adjustFec);
|
||||
|
||||
// Keep track of previous values of network state:
|
||||
// adjustment may be also based on pattern of changes in network state
|
||||
_prevTotalRate = totalRate;
|
||||
_prevRttTime = rttTime;
|
||||
_prevPacketLoss = packetLoss;
|
||||
|
||||
return codeRateDeltaAdjust;
|
||||
}
|
||||
|
||||
void
|
||||
VCMQmSelect::UpdateContent(const VideoContentMetrics* contentMetrics)
|
||||
{
|
||||
_contentMetrics = contentMetrics;
|
||||
}
|
||||
|
||||
// Select the resolution factors: frame size and frame rate change: (QM modes)
|
||||
// Selection is for going back up in resolution, or going down in.
|
||||
WebRtc_Word32
|
||||
VCMQmSelect::SelectQuality(VCMQualityMode** qm)
|
||||
VCMQmResolution::SelectResolution(VCMResolutionScale** qm)
|
||||
{
|
||||
if (!_init)
|
||||
{
|
||||
@ -272,15 +401,11 @@ VCMQmSelect::SelectQuality(VCMQualityMode** qm)
|
||||
_qm->spatialHeightFact = 1;
|
||||
_qm->temporalFact = 1;
|
||||
|
||||
|
||||
// Update native values
|
||||
_nativeWidth = _contentMetrics->nativeWidth;
|
||||
_nativeHeight = _contentMetrics->nativeHeight;
|
||||
_nativeFrameRate = _contentMetrics->nativeFrameRate;
|
||||
|
||||
// Aspect ratio: used for selection of 1x2,2x1,2x2
|
||||
_aspectRatio = (float)_width / (float)_height;
|
||||
|
||||
float avgTargetRate = 0.0f;
|
||||
float avgIncomingFrameRate = 0.0f;
|
||||
float ratioBufferLow = 0.0f;
|
||||
@ -317,13 +442,12 @@ VCMQmSelect::SelectQuality(VCMQualityMode** qm)
|
||||
// for up-sampled spatial dimensions.
|
||||
// This is needed to get the transRate for going back up in
|
||||
// spatial resolution (only 2x2 allowed in this version).
|
||||
SetMaxRateForQM(2 * _width, 2 * _height);
|
||||
WebRtc_UWord8 imageType2 = _imageType;
|
||||
WebRtc_UWord32 maxRateQM2 = _maxRateQM;
|
||||
WebRtc_UWord8 imageType2 = GetImageType(2 * _width, 2 * _height);
|
||||
WebRtc_UWord32 maxRateQM2 = kMaxRateQm[imageType2];
|
||||
|
||||
// Set the maximum transitional rate and image type:
|
||||
// for the encoder spatial dimensions.
|
||||
SetMaxRateForQM(_width, _height);
|
||||
WebRtc_UWord32 maxRateQM = kMaxRateQm[_imageType];
|
||||
|
||||
// Compute class state of the content.
|
||||
MotionNFD();
|
||||
@ -342,18 +466,21 @@ VCMQmSelect::SelectQuality(VCMQualityMode** qm)
|
||||
|
||||
// Get image class and content class: for going up spatially
|
||||
WebRtc_UWord8 imageClass2 = 1;
|
||||
if (imageType2 <= 3) imageClass2 = 0;
|
||||
if (imageType2 <= 3)
|
||||
{
|
||||
imageClass2 = 0;
|
||||
}
|
||||
WebRtc_UWord8 tableIndex2 = imageClass2 * 9 + contentClass;
|
||||
float scaleTransRate2 = kScaleTransRateQm[tableIndex2];
|
||||
|
||||
// Transitonal rate for going down
|
||||
WebRtc_UWord32 estimatedTransRateDown = static_cast<WebRtc_UWord32>
|
||||
(_incomingFrameRate * scaleTransRate * _maxRateQM / 30);
|
||||
(_incomingFrameRate * scaleTransRate * maxRateQM / 30);
|
||||
|
||||
// Transitional rate for going up temporally
|
||||
WebRtc_UWord32 estimatedTransRateUpT = static_cast<WebRtc_UWord32>
|
||||
(TRANS_RATE_SCALE_UP_TEMP * 2 * _incomingFrameRate *
|
||||
scaleTransRate * _maxRateQM / 30);
|
||||
scaleTransRate * maxRateQM / 30);
|
||||
|
||||
// Transitional rate for going up spatially
|
||||
WebRtc_UWord32 estimatedTransRateUpS = static_cast<WebRtc_UWord32>
|
||||
@ -530,7 +657,7 @@ VCMQmSelect::SelectQuality(VCMQualityMode** qm)
|
||||
}
|
||||
|
||||
WebRtc_Word32
|
||||
VCMQmSelect::SelectSpatialDirectionMode(float transRate)
|
||||
VCMQmResolution::SelectSpatialDirectionMode(float transRate)
|
||||
{
|
||||
// Default is 1x2 (H)
|
||||
|
||||
@ -579,156 +706,82 @@ VCMQmSelect::SelectSpatialDirectionMode(float transRate)
|
||||
return VCM_OK;
|
||||
}
|
||||
|
||||
void
|
||||
VCMQmSelect::Coherence()
|
||||
// DONE WITH RESOLUTION CLASS
|
||||
|
||||
|
||||
// ROBUSTNESS CLASS
|
||||
|
||||
VCMQmRobustness::VCMQmRobustness()
|
||||
{
|
||||
float horizNZ = _contentMetrics->motionHorizontalness;
|
||||
float distortionNZ = _contentMetrics->motionClusterDistortion;
|
||||
Reset();
|
||||
}
|
||||
|
||||
// Coherence measure: combine horizontalness with cluster distortion
|
||||
_coherence.value = COH_MAX;
|
||||
if (distortionNZ > 0.)
|
||||
{
|
||||
_coherence.value = horizNZ / distortionNZ;
|
||||
}
|
||||
_coherence.value = VCM_MIN(COH_MAX, _coherence.value);
|
||||
|
||||
if (_coherence.value < COHERENCE_THR)
|
||||
{
|
||||
_coherence.level = kLow;
|
||||
}
|
||||
else
|
||||
{
|
||||
_coherence.level = kHigh;
|
||||
}
|
||||
VCMQmRobustness::~VCMQmRobustness()
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
void
|
||||
VCMQmSelect::MotionNFD()
|
||||
VCMQmRobustness::Reset()
|
||||
{
|
||||
_motion.value = _contentMetrics->motionMagnitudeNZ;
|
||||
|
||||
// Determine motion level
|
||||
if (_motion.value < LOW_MOTION_NFD)
|
||||
{
|
||||
_motion.level = kLow;
|
||||
}
|
||||
else if (_motion.value > HIGH_MOTION_NFD)
|
||||
{
|
||||
_motion.level = kHigh;
|
||||
}
|
||||
else
|
||||
{
|
||||
_motion.level = kDefault;
|
||||
}
|
||||
|
||||
_prevTotalRate = 0.0f;
|
||||
_prevRttTime = 0;
|
||||
_prevPacketLoss = 0;
|
||||
ResetQM();
|
||||
return;
|
||||
}
|
||||
|
||||
void
|
||||
VCMQmSelect::Motion()
|
||||
// Adjust the FEC rate based on the content and the network state
|
||||
// (packet loss rate, total rate/bandwidth, round trip time).
|
||||
// Note that packetLoss here is the filtered loss value.
|
||||
WebRtc_UWord8
|
||||
VCMQmRobustness::AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate,
|
||||
float frameRate,WebRtc_UWord32 rttTime,
|
||||
WebRtc_UWord8 packetLoss)
|
||||
{
|
||||
if (_contentMetrics == NULL)
|
||||
{
|
||||
return VCM_OK;
|
||||
}
|
||||
|
||||
float sizeZeroMotion = _contentMetrics->sizeZeroMotion;
|
||||
float motionMagNZ = _contentMetrics->motionMagnitudeNZ;
|
||||
// Default: no adjustment
|
||||
WebRtc_UWord8 codeRateDeltaAdjust = codeRateDelta;
|
||||
float adjustFec = 1.0f;
|
||||
|
||||
// Take product of size and magnitude with equal weight
|
||||
_motion.value = (1.0f - sizeZeroMotion) * motionMagNZ;
|
||||
// Compute class state of the content.
|
||||
MotionNFD();
|
||||
Spatial();
|
||||
|
||||
// Stabilize: motionMagNZ could be large when only a
|
||||
// few motion blocks are non-zero
|
||||
_stationaryMotion = false;
|
||||
if (sizeZeroMotion > HIGH_ZERO_MOTION_SIZE)
|
||||
{
|
||||
_motion.value = 0.0f;
|
||||
_stationaryMotion = true;
|
||||
}
|
||||
// Determine motion level
|
||||
if (_motion.value < LOW_MOTION)
|
||||
{
|
||||
_motion.level = kLow;
|
||||
}
|
||||
else if (_motion.value > HIGH_MOTION)
|
||||
{
|
||||
_motion.level = kHigh;
|
||||
}
|
||||
else
|
||||
{
|
||||
_motion.level = kDefault;
|
||||
}
|
||||
// TODO (marpan):
|
||||
// Set FEC adjustment factor
|
||||
|
||||
codeRateDeltaAdjust = static_cast<WebRtc_UWord8>(codeRateDelta * adjustFec);
|
||||
|
||||
// Keep track of previous values of network state:
|
||||
// adjustment may be also based on pattern of changes in network state
|
||||
_prevTotalRate = totalRate;
|
||||
_prevRttTime = rttTime;
|
||||
_prevPacketLoss = packetLoss;
|
||||
|
||||
_prevCodeRateDelta = codeRateDelta;
|
||||
|
||||
return codeRateDeltaAdjust;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
VCMQmSelect::Spatial()
|
||||
// Set the UEP (unequal-protection) on/off for the FEC
|
||||
bool
|
||||
VCMQmRobustness::SetUepProtection(WebRtc_UWord8 codeRateDelta, float totalRate,
|
||||
WebRtc_UWord8 packetLoss, bool frameType)
|
||||
{
|
||||
float spatialErr = _contentMetrics->spatialPredErr;
|
||||
float spatialErrH = _contentMetrics->spatialPredErrH;
|
||||
float spatialErrV = _contentMetrics->spatialPredErrV;
|
||||
// Spatial measure: take average of 3 prediction errors
|
||||
_spatial.value = (spatialErr + spatialErrH + spatialErrV) / 3.0f;
|
||||
|
||||
float scale = 1.0f;
|
||||
// Reduce thresholds for HD scenes
|
||||
if (_imageType > 3)
|
||||
if (_contentMetrics == NULL)
|
||||
{
|
||||
scale = (float)SCALE_TEXTURE_HD;
|
||||
return VCM_OK;
|
||||
}
|
||||
|
||||
if (_spatial.value > scale * HIGH_TEXTURE)
|
||||
{
|
||||
_spatial.level = kHigh;
|
||||
}
|
||||
else if (_spatial.value < scale * LOW_TEXTURE)
|
||||
{
|
||||
_spatial.level = kLow;
|
||||
}
|
||||
else
|
||||
{
|
||||
_spatial.level = kDefault;
|
||||
}
|
||||
}
|
||||
// Default: UEP on
|
||||
bool uepProtection = true;
|
||||
|
||||
|
||||
WebRtc_Word32
|
||||
VCMQmSelect::SetMaxRateForQM(WebRtc_UWord32 width, WebRtc_UWord32 height)
|
||||
{
|
||||
// Match image type
|
||||
WebRtc_UWord32 imageSize = width * height;
|
||||
|
||||
if (imageSize < kFrameSizeTh[0])
|
||||
{
|
||||
_imageType = 0;
|
||||
}
|
||||
else if (imageSize < kFrameSizeTh[1])
|
||||
{
|
||||
_imageType = 1;
|
||||
}
|
||||
else if (imageSize < kFrameSizeTh[2])
|
||||
{
|
||||
_imageType = 2;
|
||||
}
|
||||
else if (imageSize < kFrameSizeTh[3])
|
||||
{
|
||||
_imageType = 3;
|
||||
}
|
||||
else if (imageSize < kFrameSizeTh[4])
|
||||
{
|
||||
_imageType = 4;
|
||||
}
|
||||
else if (imageSize < kFrameSizeTh[5])
|
||||
{
|
||||
_imageType = 5;
|
||||
}
|
||||
else
|
||||
{
|
||||
_imageType = 6;
|
||||
}
|
||||
|
||||
// Set max rate based on image size
|
||||
_maxRateQM = kMaxRateQm[_imageType];
|
||||
|
||||
return VCM_OK;
|
||||
return uepProtection;
|
||||
}
|
||||
|
||||
} // end of namespace
|
||||
|
||||
@ -13,25 +13,19 @@
|
||||
|
||||
#include "typedefs.h"
|
||||
#include "common_types.h"
|
||||
/************************/
|
||||
/* Quality Modes */
|
||||
/**********************/
|
||||
/******************************************************/
|
||||
/* Quality Modes: Resolution and Robustness settings */
|
||||
/******************************************************/
|
||||
|
||||
namespace webrtc
|
||||
{
|
||||
|
||||
struct VideoContentMetrics;
|
||||
|
||||
struct VCMQualityMode
|
||||
struct VCMResolutionScale
|
||||
{
|
||||
VCMQualityMode():spatialWidthFact(1), spatialHeightFact(1),
|
||||
VCMResolutionScale(): spatialWidthFact(1), spatialHeightFact(1),
|
||||
temporalFact(1){}
|
||||
void Reset()
|
||||
{
|
||||
spatialWidthFact = 1;
|
||||
spatialHeightFact = 1;
|
||||
temporalFact = 1;
|
||||
}
|
||||
|
||||
WebRtc_UWord16 spatialWidthFact;
|
||||
WebRtc_UWord16 spatialHeightFact;
|
||||
@ -59,53 +53,20 @@ struct VCMContFeature
|
||||
VCMMagValues level;
|
||||
};
|
||||
|
||||
class VCMQmSelect
|
||||
// QmMethod class: main class for resolution and robustness settings
|
||||
|
||||
class VCMQmMethod
|
||||
{
|
||||
public:
|
||||
VCMQmSelect();
|
||||
~VCMQmSelect();
|
||||
VCMQmMethod();
|
||||
~VCMQmMethod();
|
||||
|
||||
// Initialize:
|
||||
WebRtc_Word32 Initialize(float bitRate, float userFrameRate,
|
||||
WebRtc_UWord32 width, WebRtc_UWord32 height);
|
||||
|
||||
// Allow the user to set preferences: favor frame rate/resolution
|
||||
WebRtc_Word32 SetPreferences(WebRtc_Word8 resolPref);
|
||||
|
||||
// Extract ST (spatio-temporal) QM behavior and make decision
|
||||
// Inputs: qm: Reference to the quality modes pointer
|
||||
WebRtc_Word32 SelectQuality(VCMQualityMode** qm);
|
||||
|
||||
// Update QM with actual bit rate
|
||||
// (size of the latest encoded frame) and frame type.
|
||||
void UpdateEncodedSize(WebRtc_Word64 encodedSize,
|
||||
FrameType encodedFrameType);
|
||||
|
||||
// Update QM with new bit/frame/loss rates from SetTargetRates
|
||||
void UpdateRates(float targetBitRate, float avgSentRate,
|
||||
float incomingFrameRate, WebRtc_UWord8 packetLoss);
|
||||
|
||||
// Update QM with the content metrics
|
||||
void UpdateContent(const VideoContentMetrics* contentMetrics);
|
||||
|
||||
// Adjust FEC rate based on content
|
||||
WebRtc_UWord8 AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate,
|
||||
float frameRate, WebRtc_UWord16 rttTime,
|
||||
WebRtc_UWord8 packetLoss);
|
||||
|
||||
|
||||
// Select 1x2,2x2,2x2 spatial sampling mode
|
||||
WebRtc_Word32 SelectSpatialDirectionMode(float transRate);
|
||||
|
||||
// Reset values prior to QMSelect
|
||||
// Reset values
|
||||
void ResetQM();
|
||||
virtual void Reset() = 0;
|
||||
|
||||
// Reset rate quantities and counter values after every QMSelect call
|
||||
void ResetRates();
|
||||
|
||||
// Reset all
|
||||
void Reset();
|
||||
private:
|
||||
// Update with the content metrics
|
||||
void UpdateContent(const VideoContentMetrics* contentMetrics);
|
||||
|
||||
// Compute spatial texture magnitude and level
|
||||
void Spatial();
|
||||
@ -119,36 +80,84 @@ private:
|
||||
// Compute coherence magnitude and level
|
||||
void Coherence();
|
||||
|
||||
// Set the max rate for QM selection
|
||||
WebRtc_Word32 SetMaxRateForQM(WebRtc_UWord32 width, WebRtc_UWord32 height);
|
||||
// Get the imageType (CIF, VGA, HD, etc) for the system width/height
|
||||
WebRtc_Word8 GetImageType(WebRtc_UWord32 width, WebRtc_UWord32 height);
|
||||
|
||||
// Content Data
|
||||
const VideoContentMetrics* _contentMetrics;
|
||||
const VideoContentMetrics* _contentMetrics;
|
||||
|
||||
// Encoder rate control parameters, network parameters
|
||||
// Encoder and native frame sizes, frame rate, aspect ratio, imageType
|
||||
WebRtc_UWord32 _width;
|
||||
WebRtc_UWord32 _height;
|
||||
WebRtc_UWord32 _nativeWidth;
|
||||
WebRtc_UWord32 _nativeHeight;
|
||||
WebRtc_UWord32 _nativeFrameRate;
|
||||
float _aspectRatio;
|
||||
// Image type for the current encoder system size.
|
||||
WebRtc_UWord8 _imageType;
|
||||
|
||||
// Content L/M/H values. stationary flag
|
||||
VCMContFeature _motion;
|
||||
VCMContFeature _spatial;
|
||||
VCMContFeature _coherence;
|
||||
bool _stationaryMotion;
|
||||
bool _init;
|
||||
|
||||
};
|
||||
|
||||
// Resolution settings class
|
||||
|
||||
class VCMQmResolution : public VCMQmMethod
|
||||
{
|
||||
public:
|
||||
VCMQmResolution();
|
||||
~VCMQmResolution();
|
||||
|
||||
// Reset all quantities
|
||||
virtual void Reset();
|
||||
|
||||
// Reset rate quantities and counter values after every Select Quality call
|
||||
void ResetRates();
|
||||
|
||||
// Initialize rate control quantities after re-init of encoder.
|
||||
WebRtc_Word32 Initialize(float bitRate, float userFrameRate,
|
||||
WebRtc_UWord32 width, WebRtc_UWord32 height);
|
||||
|
||||
// Update QM with actual bit rate (size of the latest encoded frame)
|
||||
// and frame type, after every encoded frame.
|
||||
void UpdateEncodedSize(WebRtc_Word64 encodedSize,
|
||||
FrameType encodedFrameType);
|
||||
|
||||
// Update QM with new bit/frame/loss rates every ~1 sec from SetTargetRates
|
||||
void UpdateRates(float targetBitRate, float avgSentRate,
|
||||
float incomingFrameRate, WebRtc_UWord8 packetLoss);
|
||||
|
||||
// Extract ST (spatio-temporal) QM behavior and make decision
|
||||
// Inputs: qm: Reference to the quality modes pointer
|
||||
// Output: the spatial and/or temporal scale change
|
||||
WebRtc_Word32 SelectResolution(VCMResolutionScale** qm);
|
||||
|
||||
// Select 1x2,2x2,2x2 spatial sampling mode
|
||||
WebRtc_Word32 SelectSpatialDirectionMode(float transRate);
|
||||
|
||||
private:
|
||||
// Encoder rate control parameter
|
||||
float _targetBitRate;
|
||||
float _userFrameRate;
|
||||
float _incomingFrameRate;
|
||||
float _perFrameBandwidth;
|
||||
float _bufferLevel;
|
||||
|
||||
// Data accumulated every ~1sec from MediaOpt
|
||||
float _sumTargetRate;
|
||||
float _sumIncomingFrameRate;
|
||||
float _sumSeqRateMM;
|
||||
float _sumFrameRateMM;
|
||||
float _sumPacketLoss;
|
||||
float _prevTotalRate;
|
||||
WebRtc_UWord16 _prevRttTime;
|
||||
WebRtc_UWord8 _prevPacketLoss;
|
||||
WebRtc_Word64 _sumEncodedBytes;
|
||||
|
||||
// Encoder and native frame sizes
|
||||
WebRtc_UWord32 _width;
|
||||
WebRtc_UWord32 _height;
|
||||
WebRtc_UWord32 _nativeWidth;
|
||||
WebRtc_UWord32 _nativeHeight;
|
||||
// Resolution state parameters
|
||||
WebRtc_UWord8 _stateDecFactorSpatial;
|
||||
|
||||
WebRtc_UWord32 _nativeFrameRate;
|
||||
WebRtc_UWord8 _stateDecFactorTemp;
|
||||
|
||||
// Counters
|
||||
@ -157,24 +166,36 @@ private:
|
||||
WebRtc_UWord32 _updateRateCnt;
|
||||
WebRtc_UWord32 _lowBufferCnt;
|
||||
|
||||
// Content L/M/H values
|
||||
VCMContFeature _motion;
|
||||
VCMContFeature _spatial;
|
||||
VCMContFeature _coherence;
|
||||
bool _stationaryMotion;
|
||||
VCMResolutionScale* _qm;
|
||||
};
|
||||
|
||||
// Aspect ratio
|
||||
float _aspectRatio;
|
||||
// Robustness settings class
|
||||
|
||||
// Max rate to saturate the transitionalRate
|
||||
WebRtc_UWord32 _maxRateQM;
|
||||
WebRtc_UWord8 _imageType;
|
||||
class VCMQmRobustness : public VCMQmMethod
|
||||
{
|
||||
public:
|
||||
VCMQmRobustness();
|
||||
~VCMQmRobustness();
|
||||
|
||||
// User preference for resolution or qmax change
|
||||
WebRtc_UWord8 _userResolutionPref;
|
||||
bool _init;
|
||||
VCMQualityMode* _qm;
|
||||
virtual void Reset();
|
||||
|
||||
// Adjust FEC rate based on content: every ~1 sec from SetTargetRates
|
||||
WebRtc_UWord8 AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate,
|
||||
float frameRate, WebRtc_UWord32 rttTime,
|
||||
WebRtc_UWord8 packetLoss);
|
||||
|
||||
// Set the UEP protection on/off
|
||||
bool SetUepProtection(WebRtc_UWord8 codeRateDelta, float totalRate,
|
||||
WebRtc_UWord8 packetLoss, bool frameType);
|
||||
|
||||
private:
|
||||
// Previous state of network parameters
|
||||
float _prevTotalRate;
|
||||
WebRtc_UWord32 _prevRttTime;
|
||||
WebRtc_UWord8 _prevPacketLoss;
|
||||
|
||||
// Previous FEC rate
|
||||
WebRtc_UWord8 _prevCodeRateDelta;
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
@ -28,6 +28,9 @@ namespace webrtc
|
||||
// Initial level of buffer in secs: should corresponds to wrapper settings
|
||||
#define INIT_BUFFER_LEVEL 0.5
|
||||
|
||||
// Optimal level of buffer in secs: should corresponds to wrapper settings
|
||||
#define OPT_BUFFER_LEVEL 0.6
|
||||
|
||||
// Threshold of (max) buffer size below which we consider too low (underflow)
|
||||
#define PERC_BUFFER_THR 0.10
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user