diff --git a/webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h b/webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h index 8b97b3b46..c2cefdd94 100644 --- a/webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h +++ b/webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h @@ -18,11 +18,11 @@ namespace webrtc { // Ratio allocation between temporal streams: // Values as required for the VP8 codec (accumulating). static const float - kVp8LayerRateAlloction[kMaxTemporalStreams][kMaxTemporalStreams] = { - {1.0f, 0, 0, 0}, // 1 layer - {0.6f, 1.0f , 0 , 0}, // 2 layers {60%, 40%} - {0.4f, 0.6f , 1.0f, 0}, // 3 layers {40%, 20%, 40%} - {0.25f, 0.4f, 0.6f, 1.0f} // 4 layers {25%, 15%, 20%, 40%} + kVp8LayerRateAlloction[kMaxTemporalStreams][kMaxTemporalStreams] = { + {1.0f, 1.0f, 1.0f, 1.0f}, // 1 layer + {0.6f, 1.0f, 1.0f, 1.0f}, // 2 layers {60%, 40%} + {0.4f, 0.6f, 1.0f, 1.0f}, // 3 layers {40%, 20%, 40%} + {0.25f, 0.4f, 0.6f, 1.0f} // 4 layers {25%, 15%, 20%, 40%} }; } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc b/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc new file mode 100644 index 000000000..ba7412bd1 --- /dev/null +++ b/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc @@ -0,0 +1,268 @@ +/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. +* +* Use of this source code is governed by a BSD-style license +* that can be found in the LICENSE file in the root of the source +* tree. An additional intellectual property rights grant can be found +* in the file PATENTS. All contributing project authors may +* be found in the AUTHORS file in the root of the source tree. +*/ + +#include +#include + +#include "vpx/vpx_encoder.h" +#include "vpx/vp8cx.h" +#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" +#include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" +#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" + +// This file implements logic to adapt the number of temporal layers based on +// input frame rate in order to avoid having the base layer being relaying at +// a below acceptable framerate. +namespace webrtc { +namespace { +enum { + kTemporalUpdateLast = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF, + + kTemporalUpdateGolden = + VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST, + + kTemporalUpdateGoldenWithoutDependency = + kTemporalUpdateGolden | VP8_EFLAG_NO_REF_GF, + + kTemporalUpdateAltref = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_LAST, + + kTemporalUpdateAltrefWithoutDependency = + kTemporalUpdateAltref | VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF, + + kTemporalUpdateNone = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY, + + kTemporalUpdateNoneNoRefAltref = kTemporalUpdateNone | VP8_EFLAG_NO_REF_ARF, + + kTemporalUpdateNoneNoRefGoldenRefAltRef = + VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY, + + kTemporalUpdateGoldenWithoutDependencyRefAltRef = + VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST, + + kTemporalUpdateLastRefAltRef = + VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF, + + kTemporalUpdateGoldenRefAltRef = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST, + + kTemporalUpdateLastAndGoldenRefAltRef = + VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF, + + kTemporalUpdateLastRefAll = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_GF, +}; + +int CalculateNumberOfTemporalLayers(int current_temporal_layers, + int input_fps) { + if (input_fps >= 24) { + return 3; + } + if (input_fps >= 20 && current_temporal_layers >= 3) { + // Keep doing 3 temporal layers until we go below 20fps. + return 3; + } + if (input_fps >= 10) { + return 2; + } + if (input_fps > 8 && current_temporal_layers >= 2) { + // keep doing 2 temporal layers until we go below 8fps + return 2; + } + return 1; +} + +class RealTimeTemporalLayers : public TemporalLayers { + public: + RealTimeTemporalLayers(int max_num_temporal_layers, + uint8_t initial_tl0_pic_idx) + : temporal_layers_(1), + max_temporal_layers_(max_num_temporal_layers), + tl0_pic_idx_(initial_tl0_pic_idx), + frame_counter_(static_cast(-1)), + timestamp_(0), + last_base_layer_sync_(0), + layer_ids_length_(0), + layer_ids_(NULL), + encode_flags_length_(0), + encode_flags_(NULL) { + assert(max_temporal_layers_ >= 1); + assert(max_temporal_layers_ <= 3); + } + + virtual ~RealTimeTemporalLayers() {} + + virtual bool ConfigureBitrates(int bitrate_kbit, + int max_bitrate_kbit, + int framerate, + vpx_codec_enc_cfg_t* cfg) { + temporal_layers_ = + CalculateNumberOfTemporalLayers(temporal_layers_, framerate); + temporal_layers_ = std::min(temporal_layers_, max_temporal_layers_); + assert(temporal_layers_ >= 1 && temporal_layers_ <= 3); + + cfg->ts_number_layers = temporal_layers_; + for (int tl = 0; tl < temporal_layers_; ++tl) { + cfg->ts_target_bitrate[tl] = + bitrate_kbit * kVp8LayerRateAlloction[temporal_layers_ - 1][tl]; + } + + switch (temporal_layers_) { + case 1: { + static const unsigned int layer_ids[] = {0u}; + layer_ids_ = layer_ids; + layer_ids_length_ = sizeof(layer_ids) / sizeof(*layer_ids); + + static const int encode_flags[] = {kTemporalUpdateLastRefAll}; + encode_flags_length_ = sizeof(encode_flags) / sizeof(*layer_ids); + encode_flags_ = encode_flags; + + cfg->ts_rate_decimator[0] = 1; + cfg->ts_periodicity = layer_ids_length_; + } break; + + case 2: { + static const unsigned int layer_ids[] = {0u, 1u}; + layer_ids_ = layer_ids; + layer_ids_length_ = sizeof(layer_ids) / sizeof(*layer_ids); + + static const int encode_flags[] = { + kTemporalUpdateLastAndGoldenRefAltRef, + kTemporalUpdateGoldenWithoutDependencyRefAltRef, + kTemporalUpdateLastRefAltRef, kTemporalUpdateGoldenRefAltRef, + kTemporalUpdateLastRefAltRef, kTemporalUpdateGoldenRefAltRef, + kTemporalUpdateLastRefAltRef, kTemporalUpdateNone + }; + encode_flags_length_ = sizeof(encode_flags) / sizeof(*layer_ids); + encode_flags_ = encode_flags; + + cfg->ts_rate_decimator[0] = 2; + cfg->ts_rate_decimator[1] = 1; + cfg->ts_periodicity = layer_ids_length_; + } break; + + case 3: { + static const unsigned int layer_ids[] = {0u, 2u, 1u, 2u}; + layer_ids_ = layer_ids; + layer_ids_length_ = sizeof(layer_ids) / sizeof(*layer_ids); + + static const int encode_flags[] = { + kTemporalUpdateLastAndGoldenRefAltRef, + kTemporalUpdateNoneNoRefGoldenRefAltRef, + kTemporalUpdateGoldenWithoutDependencyRefAltRef, kTemporalUpdateNone, + kTemporalUpdateLastRefAltRef, kTemporalUpdateNone, + kTemporalUpdateGoldenRefAltRef, kTemporalUpdateNone + }; + encode_flags_length_ = sizeof(encode_flags) / sizeof(*layer_ids); + encode_flags_ = encode_flags; + + cfg->ts_rate_decimator[0] = 4; + cfg->ts_rate_decimator[1] = 2; + cfg->ts_rate_decimator[2] = 1; + cfg->ts_periodicity = layer_ids_length_; + } break; + + default: + assert(false); + return false; + } + memcpy( + cfg->ts_layer_id, layer_ids_, sizeof(unsigned int) * layer_ids_length_); + return true; + } + + virtual int EncodeFlags(uint32_t timestamp) { + frame_counter_++; + return CurrentEncodeFlags(); + } + + int CurrentEncodeFlags() const { + assert(encode_flags_length_ > 0 && encode_flags_ != NULL); + int index = frame_counter_ % encode_flags_length_; + assert(index >= 0 && index < encode_flags_length_); + return encode_flags_[index]; + } + + unsigned int CurrentLayerId() const { + assert(layer_ids_length_ > 0 && layer_ids_ != NULL); + int index = frame_counter_ % layer_ids_length_; + assert(index >= 0 && index < layer_ids_length_); + return layer_ids_[index]; + } + + virtual void PopulateCodecSpecific(bool base_layer_sync, + CodecSpecificInfoVP8* vp8_info, + uint32_t timestamp) { + assert(temporal_layers_ > 0); + + if (temporal_layers_ == 1) { + vp8_info->temporalIdx = kNoTemporalIdx; + vp8_info->layerSync = false; + vp8_info->tl0PicIdx = kNoTl0PicIdx; + } else { + if (base_layer_sync) { + vp8_info->temporalIdx = 0; + vp8_info->layerSync = true; + } else { + vp8_info->temporalIdx = CurrentLayerId(); + int temporal_reference = CurrentEncodeFlags(); + + if (temporal_reference == kTemporalUpdateAltrefWithoutDependency || + temporal_reference == kTemporalUpdateGoldenWithoutDependency || + temporal_reference == + kTemporalUpdateGoldenWithoutDependencyRefAltRef || + temporal_reference == kTemporalUpdateNoneNoRefGoldenRefAltRef || + (temporal_reference == kTemporalUpdateNone && + temporal_layers_ == 4)) { + vp8_info->layerSync = true; + } else { + vp8_info->layerSync = false; + } + } + if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) { + // Regardless of pattern the frame after a base layer sync will always + // be a layer sync. + vp8_info->layerSync = true; + } + if (vp8_info->temporalIdx == 0 && timestamp != timestamp_) { + timestamp_ = timestamp; + tl0_pic_idx_++; + } + last_base_layer_sync_ = base_layer_sync; + vp8_info->tl0PicIdx = tl0_pic_idx_; + } + } + + void FrameEncoded(unsigned int size, uint32_t timestamp) {} + + private: + int temporal_layers_; + int max_temporal_layers_; + + int tl0_pic_idx_; + unsigned int frame_counter_; + uint32_t timestamp_; + bool last_base_layer_sync_; + + // Pattern of temporal layer ids. + int layer_ids_length_; + const unsigned int* layer_ids_; + + // Pattern of encode flags. + int encode_flags_length_; + const int* encode_flags_; +}; +} // namespace + +TemporalLayers* RealTimeTemporalLayersFactory::Create( + int max_temporal_layers, + uint8_t initial_tl0_pic_idx) const { + return new RealTimeTemporalLayers(max_temporal_layers, initial_tl0_pic_idx); +} +} // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h index 9df88b9be..4549e287c 100644 --- a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h @@ -24,6 +24,8 @@ struct CodecSpecificInfoVP8; class TemporalLayers { public: + // Factory for TemporalLayer strategy. Default behaviour is a fixed pattern + // of temporal layers. See default_temporal_layers.cc struct Factory { Factory() {} virtual ~Factory() {} @@ -49,5 +51,14 @@ class TemporalLayers { virtual void FrameEncoded(unsigned int size, uint32_t timestamp) = 0; }; +// Factory for a temporal layers strategy that adaptively changes the number of +// layers based on input framerate so that the base layer has an acceptable +// framerate. See realtime_temporal_layers.cc +struct RealTimeTemporalLayersFactory : TemporalLayers::Factory { + virtual ~RealTimeTemporalLayersFactory() {} + virtual TemporalLayers* Create(int num_temporal_layers, + uint8_t initial_tl0_pic_idx) const; +}; + } // namespace webrtc #endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_ diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8.gyp b/webrtc/modules/video_coding/codecs/vp8/vp8.gyp index 2fd31508f..b3859a597 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8.gyp +++ b/webrtc/modules/video_coding/codecs/vp8/vp8.gyp @@ -34,6 +34,7 @@ 'vp8_impl.cc', 'default_temporal_layers.cc', 'default_temporal_layers.h', + 'realtime_temporal_layers.cc', 'temporal_layers.h', ], # Disable warnings to enable Win64 build, issue 1323. diff --git a/webrtc/modules/video_coding/main/source/video_sender_unittest.cc b/webrtc/modules/video_coding/main/source/video_sender_unittest.cc index 9f341b159..bdf294725 100644 --- a/webrtc/modules/video_coding/main/source/video_sender_unittest.cc +++ b/webrtc/modules/video_coding/main/source/video_sender_unittest.cc @@ -11,9 +11,11 @@ #include #include "testing/gtest/include/gtest/gtest.h" +#include "webrtc/common.h" #include "webrtc/common_video/test/frame_generator.h" #include "webrtc/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" +#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/modules/video_coding/main/interface/mock/mock_vcm_callbacks.h" #include "webrtc/modules/video_coding/main/interface/video_coding.h" #include "webrtc/modules/video_coding/main/source/video_coding_impl.h" @@ -31,12 +33,40 @@ using ::testing::Field; using ::testing::NiceMock; using ::testing::Pointee; using ::testing::Return; +using ::testing::FloatEq; using std::vector; using webrtc::test::FrameGenerator; namespace webrtc { namespace vcm { namespace { +enum { + kMaxNumberOfTemporalLayers = 3 +}; + +struct Vp8StreamInfo { + float framerate_fps[kMaxNumberOfTemporalLayers]; + int bitrate_kbps[kMaxNumberOfTemporalLayers]; +}; + +MATCHER_P(MatchesVp8StreamInfo, expected, "") { + bool res = true; + for (int tl = 0; tl < kMaxNumberOfTemporalLayers; ++tl) { + if (abs(expected.framerate_fps[tl] - arg.framerate_fps[tl]) > 0.5) { + *result_listener << " framerate_fps[" << tl + << "] = " << arg.framerate_fps[tl] << " (expected " + << expected.framerate_fps[tl] << ") "; + res = false; + } + if (abs(expected.bitrate_kbps[tl] - arg.bitrate_kbps[tl]) > 10) { + *result_listener << " bitrate_kbps[" << tl + << "] = " << arg.bitrate_kbps[tl] << " (expected " + << expected.bitrate_kbps[tl] << ") "; + res = false; + } + } + return res; +} class EmptyFrameGenerator : public FrameGenerator { public: @@ -81,6 +111,15 @@ class PacketizationCallback : public VCMPacketizationCallback { interval_ms(); } + Vp8StreamInfo CalculateVp8StreamInfo() { + Vp8StreamInfo info; + for (int tl = 0; tl < 3; ++tl) { + info.framerate_fps[tl] = FramerateFpsWithinTemporalLayer(tl); + info.bitrate_kbps[tl] = BitrateKbpsWithinTemporalLayer(tl); + } + return info; + } + private: struct FrameData { FrameData() {} @@ -319,6 +358,18 @@ class TestVideoSenderWithVp8 : public TestVideoSender { } } + Vp8StreamInfo SimulateWithFramerate(float framerate) { + const float short_simulation_interval = 5.0; + const float long_simulation_interval = 10.0; + // It appears that this 5 seconds simulation is needed to allow + // bitrate and framerate to stabilize. + InsertFrames(framerate, short_simulation_interval); + packetization_callback_.Reset(); + + InsertFrames(framerate, long_simulation_interval); + return packetization_callback_.CalculateVp8StreamInfo(); + } + protected: VideoCodec codec_; int codec_bitrate_kbps_; @@ -327,51 +378,56 @@ class TestVideoSenderWithVp8 : public TestVideoSender { TEST_F(TestVideoSenderWithVp8, DISABLED_ON_ANDROID(FixedTemporalLayersStrategy)) { - // It appears that this 5 seconds simulation are need to allow - // bitrate and framerate to stabilize. - // TODO(andresp): the framerate calculation should be improved. - double framerate = 30.0; - InsertFrames(framerate, 5.0); - packetization_callback_.Reset(); + const int low_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][0]; + const int mid_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][1]; + const int high_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][2]; + { + Vp8StreamInfo expected = {{7.5, 15.0, 30.0}, {low_b, mid_b, high_b}}; + EXPECT_THAT(SimulateWithFramerate(30.0), MatchesVp8StreamInfo(expected)); + } + { + Vp8StreamInfo expected = {{3.75, 7.5, 15.0}, {low_b, mid_b, high_b}}; + EXPECT_THAT(SimulateWithFramerate(15.0), MatchesVp8StreamInfo(expected)); + } +} - // Need to simulate for 10 seconds due to VP8 bitrate controller. - InsertFrames(framerate, 10.0); - EXPECT_NEAR( - packetization_callback_.FramerateFpsWithinTemporalLayer(2), 30.0, 0.5); - EXPECT_NEAR( - packetization_callback_.FramerateFpsWithinTemporalLayer(1), 15.0, 0.5); - EXPECT_NEAR( - packetization_callback_.FramerateFpsWithinTemporalLayer(0), 7.5, 0.5); - EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(2), - codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][2], - 10); - EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(1), - codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][1], - 10); - EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(0), - codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][0], - 10); +TEST_F(TestVideoSenderWithVp8, + DISABLED_ON_ANDROID(RealTimeTemporalLayersStrategy)) { + Config extra_options; + extra_options.Set( + new RealTimeTemporalLayersFactory()); + VideoCodec codec = MakeVp8VideoCodec(352, 288, 3); + codec.extra_options = &extra_options; + codec.minBitrate = 10; + codec.startBitrate = codec_bitrate_kbps_; + codec.maxBitrate = codec_bitrate_kbps_; + EXPECT_EQ(0, sender_->RegisterSendCodec(&codec, 1, 1200)); - framerate = 15.0; - InsertFrames(framerate, 5.0); - packetization_callback_.Reset(); + const int low_b = codec_bitrate_kbps_ * 0.4; + const int mid_b = codec_bitrate_kbps_ * 0.6; + const int high_b = codec_bitrate_kbps_; - InsertFrames(15.0, 10.0); - EXPECT_NEAR( - packetization_callback_.FramerateFpsWithinTemporalLayer(2), 15.0, 0.5); - EXPECT_NEAR( - packetization_callback_.FramerateFpsWithinTemporalLayer(1), 7.5, 0.5); - EXPECT_NEAR( - packetization_callback_.FramerateFpsWithinTemporalLayer(0), 3.75, 0.5); - EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(2), - codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][2], - 10); - EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(1), - codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][1], - 10); - EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(0), - codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][0], - 10); + { + Vp8StreamInfo expected = {{7.5, 15.0, 30.0}, {low_b, mid_b, high_b}}; + EXPECT_THAT(SimulateWithFramerate(30.0), MatchesVp8StreamInfo(expected)); + } + { + Vp8StreamInfo expected = {{5.0, 10.0, 20.0}, {low_b, mid_b, high_b}}; + EXPECT_THAT(SimulateWithFramerate(20.0), MatchesVp8StreamInfo(expected)); + } + { + Vp8StreamInfo expected = {{7.5, 15.0, 15.0}, {mid_b, high_b, high_b}}; + EXPECT_THAT(SimulateWithFramerate(15.0), MatchesVp8StreamInfo(expected)); + } + { + Vp8StreamInfo expected = {{5.0, 10.0, 10.0}, {mid_b, high_b, high_b}}; + EXPECT_THAT(SimulateWithFramerate(10.0), MatchesVp8StreamInfo(expected)); + } + { + // TODO(andresp): Find out why this fails with framerate = 7.5 + Vp8StreamInfo expected = {{7.0, 7.0, 7.0}, {high_b, high_b, high_b}}; + EXPECT_THAT(SimulateWithFramerate(7.0), MatchesVp8StreamInfo(expected)); + } } } // namespace } // namespace vcm