From 70e2d11ea8edf121d0cec0a10a068a9f685f1fe2 Mon Sep 17 00:00:00 2001 From: "sprang@webrtc.org" Date: Wed, 24 Sep 2014 14:06:56 +0000 Subject: [PATCH] Reduce jitter delay for low fps streams. Enabled by finch flag. BUG= R=stefan@webrtc.org Review URL: https://webrtc-codereview.appspot.com/31389005 git-svn-id: http://webrtc.googlecode.com/svn/trunk@7288 4adac7df-926f-26a2-2b94-8c16560cd09d --- webrtc/modules/modules.gyp | 1 + .../video_coding/main/source/jitter_buffer.cc | 5 +- .../main/source/jitter_estimator.cc | 161 +++++++++++++----- .../main/source/jitter_estimator.h | 19 ++- .../main/source/jitter_estimator_tests.cc | 160 +++++++++++++++++ 5 files changed, 297 insertions(+), 49 deletions(-) create mode 100644 webrtc/modules/video_coding/main/source/jitter_estimator_tests.cc diff --git a/webrtc/modules/modules.gyp b/webrtc/modules/modules.gyp index 777523aba..9650e66d0 100644 --- a/webrtc/modules/modules.gyp +++ b/webrtc/modules/modules.gyp @@ -237,6 +237,7 @@ 'video_coding/main/interface/mock/mock_vcm_callbacks.h', 'video_coding/main/source/decoding_state_unittest.cc', 'video_coding/main/source/jitter_buffer_unittest.cc', + 'video_coding/main/source/jitter_estimator_tests.cc', 'video_coding/main/source/media_optimization_unittest.cc', 'video_coding/main/source/receiver_unittest.cc', 'video_coding/main/source/session_info_unittest.cc', diff --git a/webrtc/modules/video_coding/main/source/jitter_buffer.cc b/webrtc/modules/video_coding/main/source/jitter_buffer.cc index 9aa34090e..d09fccd91 100644 --- a/webrtc/modules/video_coding/main/source/jitter_buffer.cc +++ b/webrtc/modules/video_coding/main/source/jitter_buffer.cc @@ -122,8 +122,7 @@ void FrameList::Reset(UnorderedFrameList* free_frames) { } } -VCMJitterBuffer::VCMJitterBuffer(Clock* clock, - EventFactory* event_factory) +VCMJitterBuffer::VCMJitterBuffer(Clock* clock, EventFactory* event_factory) : clock_(clock), running_(false), crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), @@ -145,7 +144,7 @@ VCMJitterBuffer::VCMJitterBuffer(Clock* clock, num_consecutive_old_frames_(0), num_consecutive_old_packets_(0), num_discarded_packets_(0), - jitter_estimate_(), + jitter_estimate_(clock), inter_frame_delay_(clock_->TimeInMilliseconds()), rtt_ms_(kDefaultRtt), nack_mode_(kNoNack), diff --git a/webrtc/modules/video_coding/main/source/jitter_estimator.cc b/webrtc/modules/video_coding/main/source/jitter_estimator.cc index 71c54a00c..b36775a4a 100644 --- a/webrtc/modules/video_coding/main/source/jitter_estimator.cc +++ b/webrtc/modules/video_coding/main/source/jitter_estimator.cc @@ -11,6 +11,8 @@ #include "webrtc/modules/video_coding/main/source/internal_defines.h" #include "webrtc/modules/video_coding/main/source/jitter_estimator.h" #include "webrtc/modules/video_coding/main/source/rtt_filter.h" +#include "webrtc/system_wrappers/interface/clock.h" +#include "webrtc/system_wrappers/interface/field_trial.h" #include #include @@ -19,7 +21,13 @@ namespace webrtc { -VCMJitterEstimator::VCMJitterEstimator(int32_t vcmId, int32_t receiverId) +enum { kStartupDelaySamples = 30 }; +enum { kFsAccuStartupSamples = 5 }; +enum { kMaxFramerateEstimate = 200 }; + +VCMJitterEstimator::VCMJitterEstimator(const Clock* clock, + int32_t vcmId, + int32_t receiverId) : _vcmId(vcmId), _receiverId(receiverId), _phi(0.97), @@ -32,8 +40,15 @@ VCMJitterEstimator::VCMJitterEstimator(int32_t vcmId, int32_t receiverId) _noiseStdDevs(2.33), // ~Less than 1% chance // (look up in normal distribution table)... _noiseStdDevOffset(30.0), // ...of getting 30 ms freezes - _rttFilter() { - Reset(); + _rttFilter(), + fps_counter_(30), // TODO(sprang): Use an estimator with limit based on + // time, rather than number of samples. + low_rate_experiment_(kInit), + clock_(clock) { + Reset(); +} + +VCMJitterEstimator::~VCMJitterEstimator() { } VCMJitterEstimator& @@ -94,6 +109,7 @@ VCMJitterEstimator::Reset() _fsCount = 0; _startupCount = 0; _rttFilter.Reset(); + fps_counter_.Reset(); } void @@ -297,35 +313,54 @@ VCMJitterEstimator::DeviationFromExpectedDelay(int64_t frameDelayMS, // Estimates the random jitter by calculating the variance of the // sample distance from the line given by theta. -void -VCMJitterEstimator::EstimateRandomJitter(double d_dT, bool incompleteFrame) -{ - double alpha; - if (_alphaCount == 0) - { - assert(_alphaCount > 0); - return; - } - alpha = static_cast(_alphaCount - 1) / static_cast(_alphaCount); - _alphaCount++; - if (_alphaCount > _alphaCountMax) - { - _alphaCount = _alphaCountMax; - } - double avgNoise = alpha * _avgNoise + (1 - alpha) * d_dT; - double varNoise = alpha * _varNoise + - (1 - alpha) * (d_dT - _avgNoise) * (d_dT - _avgNoise); - if (!incompleteFrame || varNoise > _varNoise) - { - _avgNoise = avgNoise; - _varNoise = varNoise; - } - if (_varNoise < 1.0) - { - // The variance should never be zero, since we might get - // stuck and consider all samples as outliers. - _varNoise = 1.0; +void VCMJitterEstimator::EstimateRandomJitter(double d_dT, + bool incompleteFrame) { + uint64_t now = clock_->TimeInMicroseconds(); + if (_lastUpdateT != -1) { + fps_counter_.AddSample(now - _lastUpdateT); + } + _lastUpdateT = now; + + if (_alphaCount == 0) { + assert(false); + return; + } + double alpha = + static_cast(_alphaCount - 1) / static_cast(_alphaCount); + _alphaCount++; + if (_alphaCount > _alphaCountMax) + _alphaCount = _alphaCountMax; + + if (LowRateExperimentEnabled()) { + // In order to avoid a low frame rate stream to react slower to changes, + // scale the alpha weight relative a 30 fps stream. + double fps = GetFrameRate(); + if (fps > 0.0) { + double rate_scale = 30.0 / fps; + // At startup, there can be a lot of noise in the fps estimate. + // Interpolate rate_scale linearly, from 1.0 at sample #1, to 30.0 / fps + // at sample #kStartupDelaySamples. + if (_alphaCount < kStartupDelaySamples) { + rate_scale = + (_alphaCount * rate_scale + (kStartupDelaySamples - _alphaCount)) / + kStartupDelaySamples; + } + alpha = pow(alpha, rate_scale); } + } + + double avgNoise = alpha * _avgNoise + (1 - alpha) * d_dT; + double varNoise = + alpha * _varNoise + (1 - alpha) * (d_dT - _avgNoise) * (d_dT - _avgNoise); + if (!incompleteFrame || varNoise > _varNoise) { + _avgNoise = avgNoise; + _varNoise = varNoise; + } + if (_varNoise < 1.0) { + // The variance should never be zero, since we might get + // stuck and consider all samples as outliers. + _varNoise = 1.0; + } } double @@ -387,19 +422,61 @@ VCMJitterEstimator::UpdateMaxFrameSize(uint32_t frameSizeBytes) // Returns the current filtered estimate if available, // otherwise tries to calculate an estimate. -int -VCMJitterEstimator::GetJitterEstimate(double rttMultiplier) -{ - double jitterMS = CalculateEstimate() + OPERATING_SYSTEM_JITTER; - if (_filterJitterEstimate > jitterMS) - { - jitterMS = _filterJitterEstimate; +int VCMJitterEstimator::GetJitterEstimate(double rttMultiplier) { + double jitterMS = CalculateEstimate() + OPERATING_SYSTEM_JITTER; + if (_filterJitterEstimate > jitterMS) + jitterMS = _filterJitterEstimate; + if (_nackCount >= _nackLimit) + jitterMS += _rttFilter.RttMs() * rttMultiplier; + + if (LowRateExperimentEnabled()) { + static const double kJitterScaleLowThreshold = 5.0; + static const double kJitterScaleHighThreshold = 10.0; + double fps = GetFrameRate(); + // Ignore jitter for very low fps streams. + if (fps < kJitterScaleLowThreshold) { + if (fps == 0.0) { + return jitterMS; + } + return 0; } - if (_nackCount >= _nackLimit) - { - jitterMS += _rttFilter.RttMs() * rttMultiplier; + + // Semi-low frame rate; scale by factor linearly interpolated from 0.0 at + // kJitterScaleLowThreshold to 1.0 at kJitterScaleHighThreshold. + if (fps < kJitterScaleHighThreshold) { + jitterMS = + (1.0 / (kJitterScaleHighThreshold - kJitterScaleLowThreshold)) * + (fps - kJitterScaleLowThreshold) * jitterMS; } - return static_cast(jitterMS + 0.5); + } + + return static_cast(jitterMS + 0.5); +} + +bool VCMJitterEstimator::LowRateExperimentEnabled() { + if (low_rate_experiment_ == kInit) { + std::string group = + webrtc::field_trial::FindFullName("WebRTC-ReducedJitterDelay"); + if (group == "Disabled") { + low_rate_experiment_ = kDisabled; + } else { + low_rate_experiment_ = kEnabled; + } + } + return low_rate_experiment_ == kEnabled ? true : false; +} + +double VCMJitterEstimator::GetFrameRate() const { + if (fps_counter_.count() == 0) + return 0; + + double fps = 1000000.0 / fps_counter_.ComputeMean(); + // Sanity check. + assert(fps >= 0.0); + if (fps > kMaxFramerateEstimate) { + fps = kMaxFramerateEstimate; + } + return fps; } } diff --git a/webrtc/modules/video_coding/main/source/jitter_estimator.h b/webrtc/modules/video_coding/main/source/jitter_estimator.h index dda8f8da8..ec7e35cef 100644 --- a/webrtc/modules/video_coding/main/source/jitter_estimator.h +++ b/webrtc/modules/video_coding/main/source/jitter_estimator.h @@ -11,17 +11,22 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_JITTER_ESTIMATOR_H_ #define WEBRTC_MODULES_VIDEO_CODING_JITTER_ESTIMATOR_H_ +#include "webrtc/base/rollingaccumulator.h" #include "webrtc/modules/video_coding/main/source/rtt_filter.h" #include "webrtc/typedefs.h" namespace webrtc { +class Clock; + class VCMJitterEstimator { public: - VCMJitterEstimator(int32_t vcmId = 0, int32_t receiverId = 0); - + VCMJitterEstimator(const Clock* clock, + int32_t vcmId = 0, + int32_t receiverId = 0); + virtual ~VCMJitterEstimator(); VCMJitterEstimator& operator=(const VCMJitterEstimator& rhs); // Resets the estimate to the initial state @@ -68,6 +73,8 @@ protected: double _theta[2]; // Estimated line parameters (slope, offset) double _varNoise; // Variance of the time-deviation from the line + virtual bool LowRateExperimentEnabled(); + private: // Updates the Kalman filter for the line describing // the frame size dependent jitter. @@ -109,6 +116,8 @@ private: double DeviationFromExpectedDelay(int64_t frameDelayMS, int32_t deltaFSBytes) const; + double GetFrameRate() const; + // Constants, filter parameters int32_t _vcmId; int32_t _receiverId; @@ -145,8 +154,10 @@ private: // but never goes above _nackLimit VCMRttFilter _rttFilter; - enum { kStartupDelaySamples = 30 }; - enum { kFsAccuStartupSamples = 5 }; + rtc::RollingAccumulator fps_counter_; + enum ExperimentFlag { kInit, kEnabled, kDisabled }; + ExperimentFlag low_rate_experiment_; + const Clock* clock_; }; } // namespace webrtc diff --git a/webrtc/modules/video_coding/main/source/jitter_estimator_tests.cc b/webrtc/modules/video_coding/main/source/jitter_estimator_tests.cc new file mode 100644 index 000000000..5f3475057 --- /dev/null +++ b/webrtc/modules/video_coding/main/source/jitter_estimator_tests.cc @@ -0,0 +1,160 @@ +/* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/video_coding/main/source/jitter_estimator.h" + +#include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/system_wrappers/interface/clock.h" + +namespace webrtc { + +class TestEstimator : public VCMJitterEstimator { + public: + explicit TestEstimator(bool exp_enabled) + : VCMJitterEstimator(&fake_clock_, 0, 0), + fake_clock_(0), + exp_enabled_(exp_enabled) {} + + virtual bool LowRateExperimentEnabled() { return exp_enabled_; } + + void AdvanceClock(int64_t microseconds) { + fake_clock_.AdvanceTimeMicroseconds(microseconds); + } + + private: + SimulatedClock fake_clock_; + const bool exp_enabled_; +}; + +class TestVCMJitterEstimator : public ::testing::Test { + protected: + TestVCMJitterEstimator() + : regular_estimator_(false), low_rate_estimator_(true) {} + + virtual void SetUp() { regular_estimator_.Reset(); } + + TestEstimator regular_estimator_; + TestEstimator low_rate_estimator_; +}; + +// Generates some simple test data in the form of a sawtooth wave. +class ValueGenerator { + public: + ValueGenerator(int32_t amplitude) : amplitude_(amplitude), counter_(0) {} + virtual ~ValueGenerator() {} + + int64_t Delay() { return ((counter_ % 11) - 5) * amplitude_; } + + uint32_t FrameSize() { return 1000 + Delay(); } + + void Advance() { ++counter_; } + + private: + const int32_t amplitude_; + int64_t counter_; +}; + +// 5 fps, disable jitter delay altogether. +TEST_F(TestVCMJitterEstimator, TestLowRate) { + ValueGenerator gen(10); + uint64_t time_delta = 1000000 / 5; + for (int i = 0; i < 60; ++i) { + regular_estimator_.UpdateEstimate(gen.Delay(), gen.FrameSize()); + regular_estimator_.AdvanceClock(time_delta); + low_rate_estimator_.UpdateEstimate(gen.Delay(), gen.FrameSize()); + low_rate_estimator_.AdvanceClock(time_delta); + EXPECT_GT(regular_estimator_.GetJitterEstimate(0), 0); + if (i > 2) + EXPECT_EQ(low_rate_estimator_.GetJitterEstimate(0), 0); + gen.Advance(); + } +} + +// 8 fps, steady state estimate should be in interpolated interval between 0 +// and value of previous method. +TEST_F(TestVCMJitterEstimator, TestMidRate) { + ValueGenerator gen(10); + uint64_t time_delta = 1000000 / 8; + for (int i = 0; i < 60; ++i) { + regular_estimator_.UpdateEstimate(gen.Delay(), gen.FrameSize()); + regular_estimator_.AdvanceClock(time_delta); + low_rate_estimator_.UpdateEstimate(gen.Delay(), gen.FrameSize()); + low_rate_estimator_.AdvanceClock(time_delta); + EXPECT_GT(regular_estimator_.GetJitterEstimate(0), 0); + EXPECT_GT(low_rate_estimator_.GetJitterEstimate(0), 0); + EXPECT_GE(regular_estimator_.GetJitterEstimate(0), + low_rate_estimator_.GetJitterEstimate(0)); + gen.Advance(); + } +} + +// 30 fps, steady state estimate should be same as previous method. +TEST_F(TestVCMJitterEstimator, TestHighRate) { + ValueGenerator gen(10); + uint64_t time_delta = 1000000 / 30; + for (int i = 0; i < 60; ++i) { + regular_estimator_.UpdateEstimate(gen.Delay(), gen.FrameSize()); + regular_estimator_.AdvanceClock(time_delta); + low_rate_estimator_.UpdateEstimate(gen.Delay(), gen.FrameSize()); + low_rate_estimator_.AdvanceClock(time_delta); + EXPECT_EQ(regular_estimator_.GetJitterEstimate(0), + low_rate_estimator_.GetJitterEstimate(0)); + gen.Advance(); + } +} + +// 10 fps, high jitter then low jitter. Low rate estimator should converge +// faster to low noise estimate. +TEST_F(TestVCMJitterEstimator, TestConvergence) { + // Reach a steady state with high noise. + ValueGenerator gen(50); + uint64_t time_delta = 1000000 / 10; + for (int i = 0; i < 100; ++i) { + regular_estimator_.UpdateEstimate(gen.Delay(), gen.FrameSize()); + regular_estimator_.AdvanceClock(time_delta * 2); + low_rate_estimator_.UpdateEstimate(gen.Delay(), gen.FrameSize()); + low_rate_estimator_.AdvanceClock(time_delta * 2); + gen.Advance(); + } + + int threshold = regular_estimator_.GetJitterEstimate(0) / 2; + + // New generator with zero noise. + ValueGenerator low_gen(0); + int regular_iterations = 0; + int low_rate_iterations = 0; + for (int i = 0; i < 500; ++i) { + if (regular_iterations == 0) { + regular_estimator_.UpdateEstimate(low_gen.Delay(), low_gen.FrameSize()); + regular_estimator_.AdvanceClock(time_delta); + if (regular_estimator_.GetJitterEstimate(0) < threshold) { + regular_iterations = i; + } + } + + if (low_rate_iterations == 0) { + low_rate_estimator_.UpdateEstimate(low_gen.Delay(), low_gen.FrameSize()); + low_rate_estimator_.AdvanceClock(time_delta); + if (low_rate_estimator_.GetJitterEstimate(0) < threshold) { + low_rate_iterations = i; + } + } + + if (regular_iterations != 0 && low_rate_iterations != 0) { + break; + } + + gen.Advance(); + } + + EXPECT_NE(regular_iterations, 0); + EXPECT_NE(low_rate_iterations, 0); + EXPECT_LE(low_rate_iterations, regular_iterations); +} +}