From 2c4c9148191a10c0e82c9a209d454c6b1ebbaf20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Spr=C3=A5ng?= Date: Wed, 24 Jun 2015 11:24:44 +0200 Subject: [PATCH] In screenshare mode, suppress VP8 bitrate overshoot and increase quality This change includes several improvements: * VP8 configured with new rate control * Detection of frame dropping, with qp bump for next frame * Increased target and TL0 bitrates * Reworked rate control (TL allocation) in screenshare_layers A note on performance: PSNR and SSIM is expected to get slightly worse with this cl. Frame drops and delays should however improve. BUG=4171 R=pbos@webrtc.org, stefan@webrtc.org Review URL: https://codereview.webrtc.org/1193513006. Cr-Commit-Position: refs/heads/master@{#9495} --- talk/media/webrtc/simulcast.cc | 2 +- .../webrtc/webrtcvideoengine2_unittest.cc | 3 +- .../codecs/interface/video_error_codes.h | 1 + .../codecs/vp8/default_temporal_layers.h | 22 +- .../codecs/vp8/realtime_temporal_layers.cc | 4 +- .../codecs/vp8/screenshare_layers.cc | 262 ++++++---- .../codecs/vp8/screenshare_layers.h | 71 ++- .../codecs/vp8/screenshare_layers_unittest.cc | 456 ++++++++++++------ .../codecs/vp8/simulcast_encoder_adapter.cc | 5 +- .../codecs/vp8/simulcast_unittest.h | 8 +- .../video_coding/codecs/vp8/temporal_layers.h | 4 +- .../video_coding/codecs/vp8/vp8_impl.cc | 43 +- .../main/source/generic_encoder.cc | 14 +- .../main/source/generic_encoder.h | 1 + webrtc/video/full_stack.cc | 4 +- webrtc/video/screenshare_loopback.cc | 4 +- 16 files changed, 610 insertions(+), 294 deletions(-) diff --git a/talk/media/webrtc/simulcast.cc b/talk/media/webrtc/simulcast.cc index 6a822b8ad..759cfef93 100755 --- a/talk/media/webrtc/simulcast.cc +++ b/talk/media/webrtc/simulcast.cc @@ -411,7 +411,7 @@ void LogSimulcastSubstreams(const webrtc::VideoCodec& codec) { static const int kScreenshareMinBitrateKbps = 50; static const int kScreenshareMaxBitrateKbps = 6000; -static const int kScreenshareDefaultTl0BitrateKbps = 100; +static const int kScreenshareDefaultTl0BitrateKbps = 200; static const int kScreenshareDefaultTl1BitrateKbps = 1000; static const char* kScreencastLayerFieldTrialName = diff --git a/talk/media/webrtc/webrtcvideoengine2_unittest.cc b/talk/media/webrtc/webrtcvideoengine2_unittest.cc index d83e8c4b9..230ca3b2b 100644 --- a/talk/media/webrtc/webrtcvideoengine2_unittest.cc +++ b/talk/media/webrtc/webrtcvideoengine2_unittest.cc @@ -1438,7 +1438,8 @@ TEST_F(WebRtcVideoChannel2Test, UsesCorrectSettingsForScreencast) { TEST_F(WebRtcVideoChannel2Test, ConferenceModeScreencastConfiguresTemporalLayer) { - static const int kConferenceScreencastTemporalBitrateBps = 100000; + static const int kConferenceScreencastTemporalBitrateBps = + ScreenshareLayerConfig::GetDefault().tl0_bitrate_kbps * 1000; VideoOptions options; options.conference_mode.Set(true); channel_->SetOptions(options); diff --git a/webrtc/modules/video_coding/codecs/interface/video_error_codes.h b/webrtc/modules/video_coding/codecs/interface/video_error_codes.h index 349a39a8e..28e5a32d4 100644 --- a/webrtc/modules/video_coding/codecs/interface/video_error_codes.h +++ b/webrtc/modules/video_coding/codecs/interface/video_error_codes.h @@ -27,5 +27,6 @@ #define WEBRTC_VIDEO_CODEC_UNINITIALIZED -7 #define WEBRTC_VIDEO_CODEC_ERR_REQUEST_SLI -12 #define WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE -13 +#define WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT -14 #endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_INTERFACE_VIDEO_ERROR_CODES_H diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h index 61f281f2b..19846ba5f 100644 --- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -24,20 +24,22 @@ class DefaultTemporalLayers : public TemporalLayers { // Returns the recommended VP8 encode flags needed. May refresh the decoder // and/or update the reference buffers. - virtual int EncodeFlags(uint32_t timestamp); + int EncodeFlags(uint32_t timestamp) override; - virtual bool ConfigureBitrates(int bitrate_kbit, - int max_bitrate_kbit, - int framerate, - vpx_codec_enc_cfg_t* cfg); + bool ConfigureBitrates(int bitrate_kbit, + int max_bitrate_kbit, + int framerate, + vpx_codec_enc_cfg_t* cfg) override; - virtual void PopulateCodecSpecific(bool base_layer_sync, - CodecSpecificInfoVP8* vp8_info, - uint32_t timestamp); + void PopulateCodecSpecific(bool base_layer_sync, + CodecSpecificInfoVP8* vp8_info, + uint32_t timestamp) override; - virtual void FrameEncoded(unsigned int size, uint32_t timestamp) {} + void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override {} - virtual int CurrentLayerId() const; + bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override { return false; } + + int CurrentLayerId() const override; private: enum TemporalReferences { diff --git a/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc b/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc index f16c75681..15b5af920 100644 --- a/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc @@ -239,7 +239,9 @@ class RealTimeTemporalLayers : public TemporalLayers { } } - void FrameEncoded(unsigned int size, uint32_t timestamp) {} + void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override {} + + bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override { return false; } private: int temporal_layers_; diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc index 63ef22781..ecaf3dd4a 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -11,32 +11,52 @@ #include +#include "webrtc/base/checks.h" #include "vpx/vpx_encoder.h" #include "vpx/vp8cx.h" #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" namespace webrtc { -enum { kOneSecond90Khz = 90000 }; +static const int kOneSecond90Khz = 90000; +static const int kMinTimeBetweenSyncs = kOneSecond90Khz * 5; +static const int kMaxTimeBetweenSyncs = kOneSecond90Khz * 10; +static const int kQpDeltaThresholdForSync = 8; const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5; const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0; +// Since this is TL0 we only allow updating and predicting from the LAST +// reference frame. +const int ScreenshareLayers::kTl0Flags = + VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF | + VP8_EFLAG_NO_REF_ARF; + +// Allow predicting from both TL0 and TL1. +const int ScreenshareLayers::kTl1Flags = + VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST; + +// Allow predicting from only TL0 to allow participants to switch to the high +// bitrate stream. This means predicting only from the LAST reference frame, but +// only updating GF to not corrupt TL0. +const int ScreenshareLayers::kTl1SyncFlags = + VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_UPD_LAST; + ScreenshareLayers::ScreenshareLayers(int num_temporal_layers, - uint8_t initial_tl0_pic_idx, - FrameDropper* tl0_frame_dropper, - FrameDropper* tl1_frame_dropper) - : tl0_frame_dropper_(tl0_frame_dropper), - tl1_frame_dropper_(tl1_frame_dropper), - number_of_temporal_layers_(num_temporal_layers), + uint8_t initial_tl0_pic_idx) + : number_of_temporal_layers_(num_temporal_layers), last_base_layer_sync_(false), tl0_pic_idx_(initial_tl0_pic_idx), - active_layer_(0), - framerate_(5), - last_sync_timestamp_(-1) { + active_layer_(-1), + last_timestamp_(-1), + last_sync_timestamp_(-1), + min_qp_(-1), + max_qp_(-1), + max_debt_bytes_(0), + frame_rate_(-1) { assert(num_temporal_layers > 0); assert(num_temporal_layers <= 2); - assert(tl0_frame_dropper && tl1_frame_dropper); } int ScreenshareLayers::CurrentLayerId() const { @@ -49,84 +69,125 @@ int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { // No flags needed for 1 layer screenshare. return 0; } - CalculateFramerate(timestamp); + + int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp); int flags = 0; - // Note that ARF on purpose isn't used in this scheme since it is allocated - // for the last key frame to make key frame caching possible. - if (tl0_frame_dropper_->DropFrame()) { - // Must drop TL0, encode TL1 instead. - if (tl1_frame_dropper_->DropFrame()) { - // Must drop both TL0 and TL1. - flags = -1; - } else { - active_layer_ = 1; - if (TimeToSync(timestamp)) { - last_sync_timestamp_ = timestamp; - // Allow predicting from only TL0 to allow participants to switch to the - // high bitrate stream. This means predicting only from the LAST - // reference frame, but only updating GF to not corrupt TL0. - flags = VP8_EFLAG_NO_REF_ARF; - flags |= VP8_EFLAG_NO_REF_GF; - flags |= VP8_EFLAG_NO_UPD_ARF; - flags |= VP8_EFLAG_NO_UPD_LAST; + + if (active_layer_ == -1 || + layers_[active_layer_].state != TemporalLayer::State::kDropped) { + if (layers_[0].debt_bytes_ > max_debt_bytes_) { + // Must drop TL0, encode TL1 instead. + if (layers_[1].debt_bytes_ > max_debt_bytes_) { + // Must drop both TL0 and TL1. + active_layer_ = -1; } else { - // Allow predicting from both TL0 and TL1. - flags = VP8_EFLAG_NO_REF_ARF; - flags |= VP8_EFLAG_NO_UPD_ARF; - flags |= VP8_EFLAG_NO_UPD_LAST; + active_layer_ = 1; } + } else { + active_layer_ = 0; } - } else { - active_layer_ = 0; - // Since this is TL0 we only allow updating and predicting from the LAST - // reference frame. - flags = VP8_EFLAG_NO_UPD_GF; - flags |= VP8_EFLAG_NO_UPD_ARF; - flags |= VP8_EFLAG_NO_REF_GF; - flags |= VP8_EFLAG_NO_REF_ARF; } + + switch (active_layer_) { + case 0: + flags = kTl0Flags; + break; + case 1: + if (TimeToSync(unwrapped_timestamp)) { + last_sync_timestamp_ = unwrapped_timestamp; + flags = kTl1SyncFlags; + } else { + flags = kTl1Flags; + } + break; + case -1: + flags = -1; + break; + default: + flags = -1; + RTC_NOTREACHED(); + } + // Make sure both frame droppers leak out bits. - tl0_frame_dropper_->Leak(framerate_); - tl1_frame_dropper_->Leak(framerate_); + int64_t ts_diff; + if (last_timestamp_ == -1) { + ts_diff = kOneSecond90Khz / (frame_rate_ <= 0 ? 5 : frame_rate_); + } else { + ts_diff = unwrapped_timestamp - last_timestamp_; + } + + layers_[0].UpdateDebt(ts_diff / 90); + layers_[1].UpdateDebt(ts_diff / 90); + last_timestamp_ = timestamp; return flags; } -bool ScreenshareLayers::ConfigureBitrates(int bitrate_kbit, - int max_bitrate_kbit, +bool ScreenshareLayers::ConfigureBitrates(int bitrate_kbps, + int max_bitrate_kbps, int framerate, vpx_codec_enc_cfg_t* cfg) { - if (framerate > 0) - framerate_ = framerate; + layers_[0].target_rate_kbps_ = bitrate_kbps; + layers_[1].target_rate_kbps_ = max_bitrate_kbps; - tl0_frame_dropper_->SetRates(bitrate_kbit, framerate_); - tl1_frame_dropper_->SetRates(max_bitrate_kbit, framerate_); + int target_bitrate_kbps = bitrate_kbps; if (cfg != nullptr) { // Calculate a codec target bitrate. This may be higher than TL0, gaining // quality at the expense of frame rate at TL0. Constraints: // - TL0 frame rate should not be less than framerate / kMaxTL0FpsReduction. // - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate. - double target_bitrate = - std::min(bitrate_kbit * kMaxTL0FpsReduction, - max_bitrate_kbit / kAcceptableTargetOvershoot); - cfg->rc_target_bitrate = - std::max(static_cast(bitrate_kbit), - static_cast(target_bitrate + 0.5)); + target_bitrate_kbps = + std::min(bitrate_kbps * kMaxTL0FpsReduction, + max_bitrate_kbps / kAcceptableTargetOvershoot); + + // Don't reconfigure qp limits during quality boost frames. + if (layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) { + min_qp_ = cfg->rc_min_quantizer; + max_qp_ = cfg->rc_max_quantizer; + // After a dropped frame, a frame with max qp will be encoded and the + // quality will then ramp up from there. To boost the speed of recovery, + // encode the next frame with lower max qp. TL0 is the most important to + // improve since the errors in this layer will propagate to TL1. + // Currently, reduce max qp by 20% for TL0 and 15% for TL1. + layers_[0].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 80) / 100); + layers_[1].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 85) / 100); + } + + cfg->rc_target_bitrate = std::max(bitrate_kbps, target_bitrate_kbps); } + int avg_frame_size = (target_bitrate_kbps * 1000) / (8 * framerate); + max_debt_bytes_ = 4 * avg_frame_size; + return true; } -void ScreenshareLayers::FrameEncoded(unsigned int size, uint32_t timestamp) { - if (active_layer_ == 0) { - tl0_frame_dropper_->Fill(size, true); +void ScreenshareLayers::FrameEncoded(unsigned int size, + uint32_t timestamp, + int qp) { + if (size == 0) { + layers_[active_layer_].state = TemporalLayer::State::kDropped; + return; + } + if (layers_[active_layer_].state == TemporalLayer::State::kDropped) { + layers_[active_layer_].state = TemporalLayer::State::kQualityBoost; + } + + if (qp != -1) + layers_[active_layer_].last_qp = qp; + + if (active_layer_ == 0) { + layers_[0].debt_bytes_ += size; + layers_[1].debt_bytes_ += size; + } else if (active_layer_ == 1) { + layers_[1].debt_bytes_ += size; } - tl1_frame_dropper_->Fill(size, true); } void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync, CodecSpecificInfoVP8 *vp8_info, uint32_t timestamp) { + int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp); if (number_of_temporal_layers_ == 1) { vp8_info->temporalIdx = kNoTemporalIdx; vp8_info->layerSync = false; @@ -135,13 +196,14 @@ void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync, vp8_info->temporalIdx = active_layer_; if (base_layer_sync) { vp8_info->temporalIdx = 0; - last_sync_timestamp_ = timestamp; + last_sync_timestamp_ = unwrapped_timestamp; } else if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) { // Regardless of pattern the frame after a base layer sync will always // be a layer sync. - last_sync_timestamp_ = timestamp; + last_sync_timestamp_ = unwrapped_timestamp; } - vp8_info->layerSync = (last_sync_timestamp_ == timestamp); + vp8_info->layerSync = last_sync_timestamp_ != -1 && + last_sync_timestamp_ == unwrapped_timestamp; if (vp8_info->temporalIdx == 0) { tl0_pic_idx_++; } @@ -150,27 +212,65 @@ void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync, } } -bool ScreenshareLayers::TimeToSync(uint32_t timestamp) const { - const uint32_t timestamp_diff = timestamp - last_sync_timestamp_; - return last_sync_timestamp_ < 0 || timestamp_diff > kOneSecond90Khz; +bool ScreenshareLayers::TimeToSync(int64_t timestamp) const { + if (active_layer_ != 1) { + RTC_NOTREACHED(); + return false; + } + DCHECK_NE(-1, layers_[0].last_qp); + if (layers_[1].last_qp == -1) { + // First frame in TL1 should only depend on TL0 since there are no + // previous frames in TL1. + return true; + } + + DCHECK_NE(-1, last_sync_timestamp_); + int64_t timestamp_diff = timestamp - last_sync_timestamp_; + if (timestamp_diff > kMaxTimeBetweenSyncs) { + // After a certain time, force a sync frame. + return true; + } else if (timestamp_diff < kMinTimeBetweenSyncs) { + // If too soon from previous sync frame, don't issue a new one. + return false; + } + // Issue a sync frame if difference in quality between TL0 and TL1 isn't too + // large. + if (layers_[0].last_qp - layers_[1].last_qp < kQpDeltaThresholdForSync) + return true; + return false; } -void ScreenshareLayers::CalculateFramerate(uint32_t timestamp) { - timestamp_list_.push_front(timestamp); - // Remove timestamps older than 1 second from the list. - uint32_t timestamp_diff = timestamp - timestamp_list_.back(); - while (timestamp_diff > kOneSecond90Khz) { - timestamp_list_.pop_back(); - timestamp_diff = timestamp - timestamp_list_.back(); +bool ScreenshareLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) { + if (max_qp_ == -1) + return false; + + // If layer is in the quality boost state (following a dropped frame), update + // the configuration with the adjusted (lower) qp and set the state back to + // normal. + unsigned int adjusted_max_qp; + if (layers_[active_layer_].state == TemporalLayer::State::kQualityBoost && + layers_[active_layer_].enhanced_max_qp != -1) { + adjusted_max_qp = layers_[active_layer_].enhanced_max_qp; + layers_[active_layer_].state = TemporalLayer::State::kNormal; + } else { + if (max_qp_ == -1) + return false; + adjusted_max_qp = max_qp_; // Set the normal max qp. } - // If we have encoded frames within the last second, that number of frames - // is a reasonable first estimate of the framerate. - framerate_ = timestamp_list_.size(); - if (timestamp_diff > 0) { - // Estimate the framerate by dividing the number of timestamp diffs with - // the sum of the timestamp diffs (with rounding). - framerate_ = (kOneSecond90Khz * (timestamp_list_.size() - 1) + - timestamp_diff / 2) / timestamp_diff; + + if (adjusted_max_qp == cfg->rc_max_quantizer) + return false; + + cfg->rc_max_quantizer = adjusted_max_qp; + return true; +} + +void ScreenshareLayers::TemporalLayer::UpdateDebt(int64_t delta_ms) { + uint32_t debt_reduction_bytes = target_rate_kbps_ * delta_ms / 8; + if (debt_reduction_bytes >= debt_bytes_) { + debt_bytes_ = 0; + } else { + debt_bytes_ -= debt_reduction_bytes; } } diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h index 0bc571ee0..90a8b1b88 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -13,6 +13,7 @@ #include "vpx/vpx_encoder.h" +#include "webrtc/base/timeutils.h" #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/modules/video_coding/utility/include/frame_dropper.h" #include "webrtc/typedefs.h" @@ -25,43 +26,73 @@ class ScreenshareLayers : public TemporalLayers { public: static const double kMaxTL0FpsReduction; static const double kAcceptableTargetOvershoot; + static const int kTl0Flags; + static const int kTl1Flags; + static const int kTl1SyncFlags; - ScreenshareLayers(int num_temporal_layers, - uint8_t initial_tl0_pic_idx, - FrameDropper* tl0_frame_dropper, - FrameDropper* tl1_frame_dropper); + ScreenshareLayers(int num_temporal_layers, uint8_t initial_tl0_pic_idx); virtual ~ScreenshareLayers() {} // Returns the recommended VP8 encode flags needed. May refresh the decoder // and/or update the reference buffers. - virtual int EncodeFlags(uint32_t timestamp); + int EncodeFlags(uint32_t timestamp) override; - virtual bool ConfigureBitrates(int bitrate_kbit, - int max_bitrate_kbit, - int framerate, - vpx_codec_enc_cfg_t* cfg); + bool ConfigureBitrates(int bitrate_kbps, + int max_bitrate_kbps, + int framerate, + vpx_codec_enc_cfg_t* cfg) override; - virtual void PopulateCodecSpecific(bool base_layer_sync, - CodecSpecificInfoVP8 *vp8_info, - uint32_t timestamp); + void PopulateCodecSpecific(bool base_layer_sync, + CodecSpecificInfoVP8* vp8_info, + uint32_t timestamp) override; - virtual void FrameEncoded(unsigned int size, uint32_t timestamp); + void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override; - virtual int CurrentLayerId() const; + int CurrentLayerId() const override; + + // Allows the layers adapter to update the encoder configuration prior to a + // frame being encoded. Return true if the configuration should be updated + // and false if now change is needed. + bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override; private: - void CalculateFramerate(uint32_t timestamp); - bool TimeToSync(uint32_t timestamp) const; + bool TimeToSync(int64_t timestamp) const; - FrameDropper* tl0_frame_dropper_; - FrameDropper* tl1_frame_dropper_; int number_of_temporal_layers_; bool last_base_layer_sync_; uint8_t tl0_pic_idx_; int active_layer_; - std::list timestamp_list_; - int framerate_; + int64_t last_timestamp_; int64_t last_sync_timestamp_; + rtc::TimestampWrapAroundHandler time_wrap_handler_; + int min_qp_; + int max_qp_; + uint32_t max_debt_bytes_; + int frame_rate_; + + static const int kMaxNumTemporalLayers = 2; + struct TemporalLayer { + TemporalLayer() + : state(State::kNormal), + enhanced_max_qp(-1), + last_qp(-1), + debt_bytes_(0), + target_rate_kbps_(0) {} + + enum class State { + kNormal, + kDropped, + kReencoded, + kQualityBoost, + } state; + + int enhanced_max_qp; + int last_qp; + uint32_t debt_bytes_; + uint32_t target_rate_kbps_; + + void UpdateDebt(int64_t delta_ms); + } layers_[kMaxNumTemporalLayers]; }; } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc index e12f9ce08..198be2a09 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc @@ -22,62 +22,19 @@ using ::testing::Return; namespace webrtc { -enum { kTimestampDelta5Fps = 90000 / 5 }; // 5 frames per second at 90 kHz. -enum { kTimestampDelta30Fps = 90000 / 30 }; // 30 frames per second at 90 kHz. -enum { kFrameSize = 2500 }; - -const int kFlagsTL0 = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | - VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF; -const int kFlagsTL1 = VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | - VP8_EFLAG_NO_UPD_LAST; -const int kFlagsTL1Sync = VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | - VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST; - -class ScreenshareLayersFT : public ScreenshareLayers { - public: - ScreenshareLayersFT(int num_temporal_layers, - uint8_t initial_tl0_pic_idx, - FrameDropper* tl0_frame_dropper, - FrameDropper* tl1_frame_dropper) - : ScreenshareLayers(num_temporal_layers, - initial_tl0_pic_idx, - tl0_frame_dropper, - tl1_frame_dropper) {} - virtual ~ScreenshareLayersFT() {} -}; +// 5 frames per second at 90 kHz. +const uint32_t kTimestampDelta5Fps = 90000 / 5; +const int kDefaultQp = 54; +const int kDefaultTl0BitrateKbps = 200; +const int kDefaultTl1BitrateKbps = 2000; +const int kFrameRate = 5; +const int kSyncPeriodSeconds = 5; +const int kMaxSyncPeriodSeconds = 10; class ScreenshareLayerTest : public ::testing::Test { protected: - void SetEncodeExpectations(bool drop_tl0, bool drop_tl1, int framerate) { - EXPECT_CALL(tl0_frame_dropper_, DropFrame()) - .Times(1) - .WillRepeatedly(Return(drop_tl0)); - if (drop_tl0) { - EXPECT_CALL(tl1_frame_dropper_, DropFrame()) - .Times(1) - .WillRepeatedly(Return(drop_tl1)); - } - EXPECT_CALL(tl0_frame_dropper_, Leak(framerate)) - .Times(1); - EXPECT_CALL(tl1_frame_dropper_, Leak(framerate)) - .Times(1); - if (drop_tl0) { - EXPECT_CALL(tl0_frame_dropper_, Fill(_, _)) - .Times(0); - if (drop_tl1) { - EXPECT_CALL(tl1_frame_dropper_, Fill(_, _)) - .Times(0); - } else { - EXPECT_CALL(tl1_frame_dropper_, Fill(kFrameSize, true)) - .Times(1); - } - } else { - EXPECT_CALL(tl0_frame_dropper_, Fill(kFrameSize, true)) - .Times(1); - EXPECT_CALL(tl1_frame_dropper_, Fill(kFrameSize, true)) - .Times(1); - } - } + ScreenshareLayerTest() : min_qp_(2), max_qp_(kDefaultQp), frame_size_(-1) {} + virtual ~ScreenshareLayerTest() {} void EncodeFrame(uint32_t timestamp, bool base_sync, @@ -85,39 +42,83 @@ class ScreenshareLayerTest : public ::testing::Test { int* flags) { *flags = layers_->EncodeFlags(timestamp); layers_->PopulateCodecSpecific(base_sync, vp8_info, timestamp); - layers_->FrameEncoded(kFrameSize, timestamp); + ASSERT_NE(-1, frame_size_); + layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); } - NiceMock tl0_frame_dropper_; - NiceMock tl1_frame_dropper_; - rtc::scoped_ptr layers_; + void ConfigureBitrates() { + vpx_codec_enc_cfg_t vpx_cfg; + memset(&vpx_cfg, 0, sizeof(vpx_codec_enc_cfg_t)); + vpx_cfg.rc_min_quantizer = min_qp_; + vpx_cfg.rc_max_quantizer = max_qp_; + EXPECT_TRUE(layers_->ConfigureBitrates( + kDefaultTl0BitrateKbps, kDefaultTl1BitrateKbps, kFrameRate, &vpx_cfg)); + frame_size_ = ((vpx_cfg.rc_target_bitrate * 1000) / 8) / kFrameRate; + } + + void WithQpLimits(int min_qp, int max_qp) { + min_qp_ = min_qp; + max_qp_ = max_qp; + } + + int RunGracePeriod() { + int flags = 0; + uint32_t timestamp = 0; + CodecSpecificInfoVP8 vp8_info; + bool got_tl0 = false; + bool got_tl1 = false; + for (int i = 0; i < 10; ++i) { + EncodeFrame(timestamp, false, &vp8_info, &flags); + timestamp += kTimestampDelta5Fps; + if (vp8_info.temporalIdx == 0) { + got_tl0 = true; + } else { + got_tl1 = true; + } + if (got_tl0 && got_tl1) + return timestamp; + } + ADD_FAILURE() << "Frames from both layers not received in time."; + return 0; + } + + int SkipUntilTl(int layer, int timestamp) { + CodecSpecificInfoVP8 vp8_info; + for (int i = 0; i < 5; ++i) { + layers_->EncodeFlags(timestamp); + timestamp += kTimestampDelta5Fps; + layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + if (vp8_info.temporalIdx != layer) { + layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + } else { + return timestamp; + } + } + ADD_FAILURE() << "Did not get a frame of TL" << layer << " in time."; + return 0; + } + + int min_qp_; + int max_qp_; + int frame_size_; + rtc::scoped_ptr layers_; }; TEST_F(ScreenshareLayerTest, 1Layer) { - layers_.reset( - new ScreenshareLayersFT(1, 0, &tl0_frame_dropper_, &tl1_frame_dropper_)); - EXPECT_TRUE(layers_->ConfigureBitrates(100, 1000, 5, NULL)); + layers_.reset(new ScreenshareLayers(1, 0)); + ConfigureBitrates(); int flags = 0; uint32_t timestamp = 0; CodecSpecificInfoVP8 vp8_info; // One layer screenshare should not use the frame dropper as all frames will // belong to the base layer. - EXPECT_CALL(tl0_frame_dropper_, DropFrame()) - .Times(0); - EXPECT_CALL(tl1_frame_dropper_, DropFrame()) - .Times(0); flags = layers_->EncodeFlags(timestamp); EXPECT_EQ(0, flags); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); EXPECT_EQ(static_cast(kNoTemporalIdx), vp8_info.temporalIdx); EXPECT_FALSE(vp8_info.layerSync); EXPECT_EQ(kNoTl0PicIdx, vp8_info.tl0PicIdx); - layers_->FrameEncoded(kFrameSize, timestamp); - - EXPECT_CALL(tl0_frame_dropper_, DropFrame()) - .Times(0); - EXPECT_CALL(tl1_frame_dropper_, DropFrame()) - .Times(0); + layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); flags = layers_->EncodeFlags(timestamp); EXPECT_EQ(0, flags); timestamp += kTimestampDelta5Fps; @@ -125,139 +126,241 @@ TEST_F(ScreenshareLayerTest, 1Layer) { EXPECT_EQ(static_cast(kNoTemporalIdx), vp8_info.temporalIdx); EXPECT_FALSE(vp8_info.layerSync); EXPECT_EQ(kNoTl0PicIdx, vp8_info.tl0PicIdx); - layers_->FrameEncoded(kFrameSize, timestamp); + layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); } TEST_F(ScreenshareLayerTest, 2Layer) { - layers_.reset( - new ScreenshareLayersFT(2, 0, &tl0_frame_dropper_, &tl1_frame_dropper_)); - EXPECT_TRUE(layers_->ConfigureBitrates(100, 1000, 5, NULL)); + layers_.reset(new ScreenshareLayers(2, 0)); + ConfigureBitrates(); int flags = 0; uint32_t timestamp = 0; uint8_t expected_tl0_idx = 0; CodecSpecificInfoVP8 vp8_info; - SetEncodeExpectations(false, false, 1); EncodeFrame(timestamp, false, &vp8_info, &flags); - EXPECT_EQ(kFlagsTL0, flags); + EXPECT_EQ(ScreenshareLayers::kTl0Flags, flags); EXPECT_EQ(0, vp8_info.temporalIdx); EXPECT_FALSE(vp8_info.layerSync); ++expected_tl0_idx; EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx); - EXPECT_CALL(tl1_frame_dropper_, SetRates(1000, 1)) - .Times(1); - EXPECT_TRUE(layers_->ConfigureBitrates(100, 1000, -1, NULL)); - // Insert 5 frames at 30 fps. All should belong to TL0. + // Insert 5 frames, cover grace period. All should be in TL0. for (int i = 0; i < 5; ++i) { - timestamp += kTimestampDelta30Fps; - // First iteration has a framerate based on a single frame, thus 1. - SetEncodeExpectations(false, false, 30); + timestamp += kTimestampDelta5Fps; EncodeFrame(timestamp, false, &vp8_info, &flags); EXPECT_EQ(0, vp8_info.temporalIdx); EXPECT_FALSE(vp8_info.layerSync); ++expected_tl0_idx; EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx); } - // Drop two frames from TL0, thus being coded in TL1. - timestamp += kTimestampDelta30Fps; - SetEncodeExpectations(true, false, 30); + + // First frame in TL0. + timestamp += kTimestampDelta5Fps; EncodeFrame(timestamp, false, &vp8_info, &flags); - EXPECT_EQ(kFlagsTL1Sync, flags); + EXPECT_EQ(ScreenshareLayers::kTl0Flags, flags); + EXPECT_EQ(0, vp8_info.temporalIdx); + EXPECT_FALSE(vp8_info.layerSync); + ++expected_tl0_idx; + EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx); + + // Drop two frames from TL0, thus being coded in TL1. + timestamp += kTimestampDelta5Fps; + EncodeFrame(timestamp, false, &vp8_info, &flags); + // First frame is sync frame. + EXPECT_EQ(ScreenshareLayers::kTl1SyncFlags, flags); EXPECT_EQ(1, vp8_info.temporalIdx); EXPECT_TRUE(vp8_info.layerSync); EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx); - timestamp += kTimestampDelta30Fps; - SetEncodeExpectations(true, false, 30); + timestamp += kTimestampDelta5Fps; EncodeFrame(timestamp, false, &vp8_info, &flags); - EXPECT_EQ(kFlagsTL1, flags); + EXPECT_EQ(ScreenshareLayers::kTl1Flags, flags); EXPECT_EQ(1, vp8_info.temporalIdx); EXPECT_FALSE(vp8_info.layerSync); EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx); } TEST_F(ScreenshareLayerTest, 2LayersPeriodicSync) { - layers_.reset( - new ScreenshareLayersFT(2, 0, &tl0_frame_dropper_, &tl1_frame_dropper_)); - EXPECT_TRUE(layers_->ConfigureBitrates(100, 1000, 5, NULL)); + layers_.reset(new ScreenshareLayers(2, 0)); + ConfigureBitrates(); int flags = 0; uint32_t timestamp = 0; CodecSpecificInfoVP8 vp8_info; - const int kNumFrames = 10; - const bool kDrops[kNumFrames] = {false, true, true, true, true, - true, true, true, true, true}; - const int kExpectedFramerates[kNumFrames] = {1, 5, 5, 5, 5, 5, 5, 5, 5, 5}; - const bool kExpectedSyncs[kNumFrames] = {false, true, false, false, false, - false, false, true, false, false}; - const int kExpectedTemporalIdx[kNumFrames] = {0, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + std::vector sync_times; + + const int kNumFrames = kSyncPeriodSeconds * kFrameRate * 2 - 1; for (int i = 0; i < kNumFrames; ++i) { timestamp += kTimestampDelta5Fps; - SetEncodeExpectations(kDrops[i], false, kExpectedFramerates[i]); EncodeFrame(timestamp, false, &vp8_info, &flags); - EXPECT_EQ(kExpectedTemporalIdx[i], vp8_info.temporalIdx); - EXPECT_EQ(kExpectedSyncs[i], vp8_info.layerSync) << "Iteration: " << i; - EXPECT_EQ(1, vp8_info.tl0PicIdx); + if (vp8_info.temporalIdx == 1 && vp8_info.layerSync) { + sync_times.push_back(timestamp); + } } + + ASSERT_EQ(2u, sync_times.size()); + EXPECT_GE(sync_times[1] - sync_times[0], 90000 * kSyncPeriodSeconds); +} + +TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) { + layers_.reset(new ScreenshareLayers(2, 0)); + ConfigureBitrates(); + uint32_t timestamp = 0; + CodecSpecificInfoVP8 vp8_info; + std::vector sync_times; + + const int kNumFrames = kMaxSyncPeriodSeconds * kFrameRate * 2 - 1; + for (int i = 0; i < kNumFrames; ++i) { + timestamp += kTimestampDelta5Fps; + layers_->EncodeFlags(timestamp); + layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + + // Simulate TL1 being at least 8 qp steps better. + if (vp8_info.temporalIdx == 0) { + layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + } else { + layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp - 8); + } + + if (vp8_info.temporalIdx == 1 && vp8_info.layerSync) + sync_times.push_back(timestamp); + } + + ASSERT_EQ(2u, sync_times.size()); + EXPECT_GE(sync_times[1] - sync_times[0], 90000 * kMaxSyncPeriodSeconds); +} + +TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) { + layers_.reset(new ScreenshareLayers(2, 0)); + ConfigureBitrates(); + uint32_t timestamp = 0; + CodecSpecificInfoVP8 vp8_info; + std::vector sync_times; + + const int kNumFrames = (kSyncPeriodSeconds + + ((kMaxSyncPeriodSeconds - kSyncPeriodSeconds) / 2)) * + kFrameRate; + for (int i = 0; i < kNumFrames; ++i) { + timestamp += kTimestampDelta5Fps; + layers_->EncodeFlags(timestamp); + layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + + // Simulate TL1 being at least 8 qp steps better. + if (vp8_info.temporalIdx == 0) { + layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + } else { + layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp - 8); + } + + if (vp8_info.temporalIdx == 1 && vp8_info.layerSync) + sync_times.push_back(timestamp); + } + + ASSERT_EQ(1u, sync_times.size()); + + bool bumped_tl0_quality = false; + for (int i = 0; i < 3; ++i) { + timestamp += kTimestampDelta5Fps; + int flags = layers_->EncodeFlags(timestamp); + layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + + if (vp8_info.temporalIdx == 0) { + // Bump TL0 to same quality as TL1. + layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp - 8); + bumped_tl0_quality = true; + } else { + layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp - 8); + if (bumped_tl0_quality) { + EXPECT_TRUE(vp8_info.layerSync); + EXPECT_EQ(ScreenshareLayers::kTl1SyncFlags, flags); + return; + } + } + } + ADD_FAILURE() << "No TL1 frame arrived within time limit."; } TEST_F(ScreenshareLayerTest, 2LayersToggling) { - layers_.reset( - new ScreenshareLayersFT(2, 0, &tl0_frame_dropper_, &tl1_frame_dropper_)); - EXPECT_TRUE(layers_->ConfigureBitrates(100, 1000, 5, NULL)); + layers_.reset(new ScreenshareLayers(2, 0)); + ConfigureBitrates(); + int flags = 0; + CodecSpecificInfoVP8 vp8_info; + uint32_t timestamp = RunGracePeriod(); + + // Insert 50 frames. 2/5 should be TL0. + int tl0_frames = 0; + int tl1_frames = 0; + for (int i = 0; i < 50; ++i) { + timestamp += kTimestampDelta5Fps; + EncodeFrame(timestamp, false, &vp8_info, &flags); + switch (vp8_info.temporalIdx) { + case 0: + ++tl0_frames; + break; + case 1: + ++tl1_frames; + break; + default: + abort(); + } + } + EXPECT_EQ(20, tl0_frames); + EXPECT_EQ(30, tl1_frames); +} + +TEST_F(ScreenshareLayerTest, AllFitsLayer0) { + layers_.reset(new ScreenshareLayers(2, 0)); + ConfigureBitrates(); + frame_size_ = ((kDefaultTl0BitrateKbps * 1000) / 8) / kFrameRate; + int flags = 0; uint32_t timestamp = 0; CodecSpecificInfoVP8 vp8_info; - const int kNumFrames = 10; - const bool kDrops[kNumFrames] = {false, true, false, true, false, - true, false, true, false, true}; - const int kExpectedFramerates[kNumFrames] = {1, 5, 5, 5, 5, 5, 5, 5, 5, 5}; - const bool kExpectedSyncs[kNumFrames] = {false, true, false, false, false, - false, false, true, false, false}; - const int kExpectedTemporalIdx[kNumFrames] = {0, 1, 0, 1, 0, 1, 0, 1, 0, 1}; - const int kExpectedTl0Idx[kNumFrames] = {1, 1, 2, 2, 3, 3, 4, 4, 5, 5}; - for (int i = 0; i < kNumFrames; ++i) { - timestamp += kTimestampDelta5Fps; - SetEncodeExpectations(kDrops[i], false, kExpectedFramerates[i]); + // Insert 50 frames, small enough that all fits in TL0. + for (int i = 0; i < 50; ++i) { EncodeFrame(timestamp, false, &vp8_info, &flags); - EXPECT_EQ(kExpectedTemporalIdx[i], vp8_info.temporalIdx); - EXPECT_EQ(kExpectedSyncs[i], vp8_info.layerSync) << "Iteration: " << i; - EXPECT_EQ(kExpectedTl0Idx[i], vp8_info.tl0PicIdx); + timestamp += kTimestampDelta5Fps; + EXPECT_EQ(ScreenshareLayers::kTl0Flags, flags); + EXPECT_EQ(0, vp8_info.temporalIdx); } } -TEST_F(ScreenshareLayerTest, 2LayersBothDrops) { - layers_.reset( - new ScreenshareLayersFT(2, 0, &tl0_frame_dropper_, &tl1_frame_dropper_)); - EXPECT_TRUE(layers_->ConfigureBitrates(100, 1000, 5, NULL)); +TEST_F(ScreenshareLayerTest, TooHighBitrate) { + layers_.reset(new ScreenshareLayers(2, 0)); + ConfigureBitrates(); + frame_size_ = 2 * ((kDefaultTl1BitrateKbps * 1000) / 8) / kFrameRate; int flags = 0; - uint32_t timestamp = 0; - uint8_t expected_tl0_idx = 0; CodecSpecificInfoVP8 vp8_info; - SetEncodeExpectations(false, false, 1); - EncodeFrame(timestamp, false, &vp8_info, &flags); - EXPECT_EQ(kFlagsTL0, flags); - EXPECT_EQ(0, vp8_info.temporalIdx); - EXPECT_FALSE(vp8_info.layerSync); - ++expected_tl0_idx; - EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx); + uint32_t timestamp = RunGracePeriod(); - timestamp += kTimestampDelta5Fps; - SetEncodeExpectations(true, false, 5); - EncodeFrame(timestamp, false, &vp8_info, &flags); - EXPECT_EQ(kFlagsTL1Sync, flags); - EXPECT_EQ(1, vp8_info.temporalIdx); - EXPECT_TRUE(vp8_info.layerSync); - EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx); + // Insert 100 frames. Half should be dropped. + int tl0_frames = 0; + int tl1_frames = 0; + int dropped_frames = 0; + for (int i = 0; i < 100; ++i) { + timestamp += kTimestampDelta5Fps; + EncodeFrame(timestamp, false, &vp8_info, &flags); + if (flags == -1) { + ++dropped_frames; + } else { + switch (vp8_info.temporalIdx) { + case 0: + ++tl0_frames; + break; + case 1: + ++tl1_frames; + break; + default: + abort(); + } + } + } - timestamp += kTimestampDelta5Fps; - SetEncodeExpectations(true, true, 5); - flags = layers_->EncodeFlags(timestamp); - EXPECT_EQ(-1, flags); + EXPECT_EQ(5, tl0_frames); + EXPECT_EQ(45, tl1_frames); + EXPECT_EQ(50, dropped_frames); } TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL0) { - layers_.reset( - new ScreenshareLayersFT(2, 0, &tl0_frame_dropper_, &tl1_frame_dropper_)); + layers_.reset(new ScreenshareLayers(2, 0)); vpx_codec_enc_cfg_t cfg; layers_->ConfigureBitrates(100, 1000, 5, &cfg); @@ -268,8 +371,7 @@ TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL0) { } TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL1) { - layers_.reset( - new ScreenshareLayersFT(2, 0, &tl0_frame_dropper_, &tl1_frame_dropper_)); + layers_.reset(new ScreenshareLayers(2, 0)); vpx_codec_enc_cfg_t cfg; layers_->ConfigureBitrates(100, 450, 5, &cfg); @@ -279,12 +381,64 @@ TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL1) { } TEST_F(ScreenshareLayerTest, TargetBitrateBelowTL0) { - layers_.reset( - new ScreenshareLayersFT(2, 0, &tl0_frame_dropper_, &tl1_frame_dropper_)); + layers_.reset(new ScreenshareLayers(2, 0)); vpx_codec_enc_cfg_t cfg; layers_->ConfigureBitrates(100, 100, 5, &cfg); EXPECT_EQ(100U, cfg.rc_target_bitrate); } +TEST_F(ScreenshareLayerTest, EncoderDrop) { + layers_.reset(new ScreenshareLayers(2, 0)); + ConfigureBitrates(); + CodecSpecificInfoVP8 vp8_info; + vpx_codec_enc_cfg_t cfg; + cfg.rc_max_quantizer = kDefaultQp; + + uint32_t timestamp = RunGracePeriod(); + timestamp = SkipUntilTl(0, timestamp); + + // Size 0 indicates dropped frame. + layers_->FrameEncoded(0, timestamp, kDefaultQp); + timestamp += kTimestampDelta5Fps; + EXPECT_FALSE(layers_->UpdateConfiguration(&cfg)); + EXPECT_EQ(ScreenshareLayers::kTl0Flags, layers_->EncodeFlags(timestamp)); + layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + + timestamp = SkipUntilTl(0, timestamp); + EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); + EXPECT_LT(cfg.rc_max_quantizer, static_cast(kDefaultQp)); + layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + + layers_->EncodeFlags(timestamp); + timestamp += kTimestampDelta5Fps; + EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); + layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + EXPECT_EQ(cfg.rc_max_quantizer, static_cast(kDefaultQp)); + layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + + // Next drop in TL1. + + timestamp = SkipUntilTl(1, timestamp); + layers_->FrameEncoded(0, timestamp, kDefaultQp); + timestamp += kTimestampDelta5Fps; + EXPECT_FALSE(layers_->UpdateConfiguration(&cfg)); + EXPECT_EQ(ScreenshareLayers::kTl1Flags, layers_->EncodeFlags(timestamp)); + layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + + timestamp = SkipUntilTl(1, timestamp); + EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); + EXPECT_LT(cfg.rc_max_quantizer, static_cast(kDefaultQp)); + layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + + layers_->EncodeFlags(timestamp); + timestamp += kTimestampDelta5Fps; + EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); + layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); + EXPECT_EQ(cfg.rc_max_quantizer, static_cast(kDefaultQp)); + layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); +} + } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc index 280296c7b..ee7fd859b 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc @@ -102,10 +102,7 @@ struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayers::Factory { virtual webrtc::TemporalLayers* Create(int num_temporal_layers, uint8_t initial_tl0_pic_idx) const { - return new webrtc::ScreenshareLayers(num_temporal_layers, - rand(), - &tl0_frame_dropper_, - &tl1_frame_dropper_); + return new webrtc::ScreenshareLayers(num_temporal_layers, rand()); } mutable webrtc::FrameDropper tl0_frame_dropper_; diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h index fffe4abdc..2e436a91f 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h @@ -196,12 +196,16 @@ class SkipEncodingUnusedStreamsTest { layers_->PopulateCodecSpecific(base_layer_sync, vp8_info, timestamp); } - void FrameEncoded(unsigned int size, uint32_t timestamp) override { - layers_->FrameEncoded(size, timestamp); + void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override { + layers_->FrameEncoded(size, timestamp, qp); } int CurrentLayerId() const override { return layers_->CurrentLayerId(); } + bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override { + return false; + } + int configured_bitrate_; TemporalLayers* layers_; }; diff --git a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h index b3b73a650..7607210d5 100644 --- a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h @@ -47,9 +47,11 @@ class TemporalLayers { CodecSpecificInfoVP8* vp8_info, uint32_t timestamp) = 0; - virtual void FrameEncoded(unsigned int size, uint32_t timestamp) = 0; + virtual void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) = 0; virtual int CurrentLayerId() const = 0; + + virtual bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) = 0; }; // Factory for a temporal layers strategy that adaptively changes the number of diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc index 437042447..05e0799b5 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -278,7 +278,6 @@ int VP8EncoderImpl::SetRates(uint32_t new_bitrate_kbit, int tl0_bitrate = std::min(codec_.targetBitrate, target_bitrate); max_bitrate = std::min(codec_.maxBitrate, target_bitrate); target_bitrate = tl0_bitrate; - framerate = -1; } configurations_[i].rc_target_bitrate = target_bitrate; temporal_layers_[stream_idx]->ConfigureBitrates(target_bitrate, @@ -312,10 +311,8 @@ void VP8EncoderImpl::SetupTemporalLayers(int num_streams, if (num_streams == 1) { if (codec.mode == kScreensharing) { // Special mode when screensharing on a single stream. - temporal_layers_.push_back(new ScreenshareLayers(num_temporal_layers, - rand(), - &tl0_frame_dropper_, - &tl1_frame_dropper_)); + temporal_layers_.push_back( + new ScreenshareLayers(num_temporal_layers, rand())); } else { temporal_layers_.push_back( tl_factory.Create(num_temporal_layers, rand())); @@ -670,8 +667,10 @@ int VP8EncoderImpl::InitAndSetControlSettings() { static_cast(token_partitions_)); vpx_codec_control(&(encoders_[i]), VP8E_SET_MAX_INTRA_BITRATE_PCT, rc_max_intra_target_); + // VP8E_SET_SCREEN_CONTENT_MODE 2 = screen content with more aggressive + // rate control (drop frames on large target bitrate overshoot) vpx_codec_control(&(encoders_[i]), VP8E_SET_SCREEN_CONTENT_MODE, - codec_.mode == kScreensharing); + codec_.mode == kScreensharing ? 2 : 0); } inited_ = true; return WEBRTC_VIDEO_CODEC_OK; @@ -698,15 +697,12 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame, const std::vector* frame_types) { TRACE_EVENT1("webrtc", "VP8::Encode", "timestamp", frame.timestamp()); - if (!inited_) { + if (!inited_) return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - } - if (frame.IsZeroSize()) { + if (frame.IsZeroSize()) return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - } - if (encoded_complete_callback_ == NULL) { + if (encoded_complete_callback_ == NULL) return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - } // Only apply scaling to improve for single-layer streams. The scaling metrics // use frame drops as a signal and is only applicable when we drop frames. @@ -851,6 +847,16 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame, // whereas |encoder_| is from highest to lowest resolution. size_t stream_idx = encoders_.size() - 1; for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) { + // Allow the layers adapter to temporarily modify the configuration. This + // change isn't stored in configurations_ so change will be discarded at + // the next update. + vpx_codec_enc_cfg_t temp_config; + memcpy(&temp_config, &configurations_[i], sizeof(vpx_codec_enc_cfg_t)); + if (temporal_layers_[stream_idx]->UpdateConfiguration(&temp_config)) { + if (vpx_codec_enc_config_set(&encoders_[i], &temp_config)) + return WEBRTC_VIDEO_CODEC_ERROR; + } + vpx_codec_control(&encoders_[i], VP8E_SET_FRAME_FLAGS, flags[stream_idx]); vpx_codec_control(&encoders_[i], VP8E_SET_TEMPORAL_LAYER_ID, @@ -873,9 +879,8 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame, vpx_codec_control(&(encoders_[0]), VP8E_SET_MAX_INTRA_BITRATE_PCT, rc_max_intra_target_); } - if (error) { + if (error) return WEBRTC_VIDEO_CODEC_ERROR; - } timestamp_ += duration; return GetEncodedPartitions(input_image, only_predict_from_key_frame); } @@ -933,6 +938,7 @@ void VP8EncoderImpl::PopulateCodecSpecific( int VP8EncoderImpl::GetEncodedPartitions(const VideoFrame& input_image, bool only_predicting_from_key_frame) { int stream_idx = static_cast(encoders_.size()) - 1; + int result = WEBRTC_VIDEO_CODEC_OK; for (size_t encoder_idx = 0; encoder_idx < encoders_.size(); ++encoder_idx, --stream_idx) { vpx_codec_iter_t iter = NULL; @@ -981,9 +987,12 @@ int VP8EncoderImpl::GetEncodedPartitions(const VideoFrame& input_image, encoded_images_[encoder_idx]._timeStamp = input_image.timestamp(); encoded_images_[encoder_idx].capture_time_ms_ = input_image.render_time_ms(); + + int qp = -1; + vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp); temporal_layers_[stream_idx]->FrameEncoded( encoded_images_[encoder_idx]._length, - encoded_images_[encoder_idx]._timeStamp); + encoded_images_[encoder_idx]._timeStamp, qp); if (send_stream_[stream_idx]) { if (encoded_images_[encoder_idx]._length > 0) { TRACE_COUNTER_ID1("webrtc", "EncodedFrameSize", encoder_idx, @@ -994,6 +1003,8 @@ int VP8EncoderImpl::GetEncodedPartitions(const VideoFrame& input_image, codec_.simulcastStream[stream_idx].width; encoded_complete_callback_->Encoded(encoded_images_[encoder_idx], &codec_specific, &frag_info); + } else if (codec_.mode == kScreensharing) { + result = WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT; } } else { // Required in case padding is applied to dropped frames. @@ -1017,7 +1028,7 @@ int VP8EncoderImpl::GetEncodedPartitions(const VideoFrame& input_image, quality_scaler_.ReportDroppedFrame(); } } - return WEBRTC_VIDEO_CODEC_OK; + return result; } int VP8EncoderImpl::SetChannelParameters(uint32_t packetLoss, int64_t rtt) { diff --git a/webrtc/modules/video_coding/main/source/generic_encoder.cc b/webrtc/modules/video_coding/main/source/generic_encoder.cc index 109150101..f2cdd599a 100644 --- a/webrtc/modules/video_coding/main/source/generic_encoder.cc +++ b/webrtc/modules/video_coding/main/source/generic_encoder.cc @@ -60,7 +60,8 @@ VCMGenericEncoder::VCMGenericEncoder(VideoEncoder* encoder, bit_rate_(0), frame_rate_(0), internal_source_(internalSource), - rotation_(kVideoRotation_0) { + rotation_(kVideoRotation_0), + is_screenshare_(false) { } VCMGenericEncoder::~VCMGenericEncoder() @@ -90,6 +91,7 @@ VCMGenericEncoder::InitEncode(const VideoCodec* settings, frame_rate_ = settings->maxFramerate; } + is_screenshare_ = settings->mode == VideoCodecMode::kScreensharing; if (encoder_->InitEncode(settings, numberOfCores, maxPayloadSize) != 0) { LOG(LS_ERROR) << "Failed to initialize the encoder associated with " "payload name: " << settings->plName; @@ -114,7 +116,15 @@ int32_t VCMGenericEncoder::Encode(const VideoFrame& inputFrame, vcm_encoded_frame_callback_->SetRotation(rotation_); } - return encoder_->Encode(inputFrame, codecSpecificInfo, &video_frame_types); + int32_t result = + encoder_->Encode(inputFrame, codecSpecificInfo, &video_frame_types); + if (is_screenshare_ && + result == WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT) { + // Target bitrate exceeded, encoder state has been reset - try again. + return encoder_->Encode(inputFrame, codecSpecificInfo, &video_frame_types); + } + + return result; } int32_t diff --git a/webrtc/modules/video_coding/main/source/generic_encoder.h b/webrtc/modules/video_coding/main/source/generic_encoder.h index a72277214..862c06ba2 100644 --- a/webrtc/modules/video_coding/main/source/generic_encoder.h +++ b/webrtc/modules/video_coding/main/source/generic_encoder.h @@ -149,6 +149,7 @@ private: const bool internal_source_; mutable rtc::CriticalSection rates_lock_; VideoRotation rotation_; + bool is_screenshare_; }; // end of VCMGenericEncoder class } // namespace webrtc diff --git a/webrtc/video/full_stack.cc b/webrtc/video/full_stack.cc index 63341b1c2..f0c5d7985 100644 --- a/webrtc/video/full_stack.cc +++ b/webrtc/video/full_stack.cc @@ -649,8 +649,8 @@ TEST_F(FullStackTest, ScreenshareSlides) { {"screenshare_slides", 1850, 1110, 5}, true, 50000, - 100000, - 1000000, + 200000, + 2000000, 0.0, 0.0, kFullStackTestDurationSecs}; diff --git a/webrtc/video/screenshare_loopback.cc b/webrtc/video/screenshare_loopback.cc index c0f96d4ff..f2133a211 100644 --- a/webrtc/video/screenshare_loopback.cc +++ b/webrtc/video/screenshare_loopback.cc @@ -45,12 +45,12 @@ size_t MinBitrate() { return static_cast(FLAGS_min_bitrate); } -DEFINE_int32(tl0_bitrate, 100, "Temporal layer 0 target bitrate."); +DEFINE_int32(tl0_bitrate, 200, "Temporal layer 0 target bitrate."); size_t StartBitrate() { return static_cast(FLAGS_tl0_bitrate); } -DEFINE_int32(tl1_bitrate, 1000, "Temporal layer 1 target bitrate."); +DEFINE_int32(tl1_bitrate, 2000, "Temporal layer 1 target bitrate."); size_t MaxBitrate() { return static_cast(FLAGS_tl1_bitrate); }