generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Adding temporal layer strategy that keeps base layer framerate at an acceptable value.

Browse Source 
R=stefan@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/2272005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@4911 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
andresp@webrtc.org 2013-10-03 14:06:14 +00:00
parent 70df305760
commit 26f78f7ecb
5 changed files with 383 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h
View File

@ -18,11 +18,11 @@ namespace webrtc {
// Ratio allocation between temporal streams:
// Values as required for the VP8 codec (accumulating).
static const float
kVp8LayerRateAlloction[kMaxTemporalStreams][kMaxTemporalStreams] = {
{1.0f, 0, 0, 0}, // 1 layer
{0.6f, 1.0f , 0 , 0}, // 2 layers {60%, 40%}
{0.4f, 0.6f , 1.0f, 0}, // 3 layers {40%, 20%, 40%}
{0.25f, 0.4f, 0.6f, 1.0f} // 4 layers {25%, 15%, 20%, 40%}
kVp8LayerRateAlloction[kMaxTemporalStreams][kMaxTemporalStreams] = {
{1.0f, 1.0f, 1.0f, 1.0f}, // 1 layer
{0.6f, 1.0f, 1.0f, 1.0f}, // 2 layers {60%, 40%}
{0.4f, 0.6f, 1.0f, 1.0f}, // 3 layers {40%, 20%, 40%}
{0.25f, 0.4f, 0.6f, 1.0f} // 4 layers {25%, 15%, 20%, 40%}
};
} // namespace webrtc

268
webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc Normal file
View File

@ -0,0 +1,268 @@
/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <stdlib.h>
#include <algorithm>
#include "vpx/vpx_encoder.h"
#include "vpx/vp8cx.h"
#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
#include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h"
#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
// This file implements logic to adapt the number of temporal layers based on
// input frame rate in order to avoid having the base layer being relaying at
// a below acceptable framerate.
namespace webrtc {
namespace {
enum {
kTemporalUpdateLast = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF,
kTemporalUpdateGolden =
VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
kTemporalUpdateGoldenWithoutDependency =
kTemporalUpdateGolden | VP8_EFLAG_NO_REF_GF,
kTemporalUpdateAltref = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_LAST,
kTemporalUpdateAltrefWithoutDependency =
kTemporalUpdateAltref | VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF,
kTemporalUpdateNone = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
kTemporalUpdateNoneNoRefAltref = kTemporalUpdateNone | VP8_EFLAG_NO_REF_ARF,
kTemporalUpdateNoneNoRefGoldenRefAltRef =
VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
kTemporalUpdateGoldenWithoutDependencyRefAltRef =
VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
kTemporalUpdateLastRefAltRef =
VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
kTemporalUpdateGoldenRefAltRef = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
kTemporalUpdateLastAndGoldenRefAltRef =
VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
kTemporalUpdateLastRefAll = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_GF,
};
int CalculateNumberOfTemporalLayers(int current_temporal_layers,
int input_fps) {
if (input_fps >= 24) {
return 3;
}
if (input_fps >= 20 && current_temporal_layers >= 3) {
// Keep doing 3 temporal layers until we go below 20fps.
return 3;
}
if (input_fps >= 10) {
return 2;
}
if (input_fps > 8 && current_temporal_layers >= 2) {
// keep doing 2 temporal layers until we go below 8fps
return 2;
}
return 1;
}
class RealTimeTemporalLayers : public TemporalLayers {
public:
RealTimeTemporalLayers(int max_num_temporal_layers,
uint8_t initial_tl0_pic_idx)
: temporal_layers_(1),
max_temporal_layers_(max_num_temporal_layers),
tl0_pic_idx_(initial_tl0_pic_idx),
frame_counter_(static_cast<unsigned int>(-1)),
timestamp_(0),
last_base_layer_sync_(0),
layer_ids_length_(0),
layer_ids_(NULL),
encode_flags_length_(0),
encode_flags_(NULL) {
assert(max_temporal_layers_ >= 1);
assert(max_temporal_layers_ <= 3);
}
virtual ~RealTimeTemporalLayers() {}
virtual bool ConfigureBitrates(int bitrate_kbit,
int max_bitrate_kbit,
int framerate,
vpx_codec_enc_cfg_t* cfg) {
temporal_layers_ =
CalculateNumberOfTemporalLayers(temporal_layers_, framerate);
temporal_layers_ = std::min(temporal_layers_, max_temporal_layers_);
assert(temporal_layers_ >= 1 && temporal_layers_ <= 3);
cfg->ts_number_layers = temporal_layers_;
for (int tl = 0; tl < temporal_layers_; ++tl) {
cfg->ts_target_bitrate[tl] =
bitrate_kbit * kVp8LayerRateAlloction[temporal_layers_ - 1][tl];
}
switch (temporal_layers_) {
case 1: {
static const unsigned int layer_ids[] = {0u};
layer_ids_ = layer_ids;
layer_ids_length_ = sizeof(layer_ids) / sizeof(*layer_ids);
static const int encode_flags[] = {kTemporalUpdateLastRefAll};
encode_flags_length_ = sizeof(encode_flags) / sizeof(*layer_ids);
encode_flags_ = encode_flags;
cfg->ts_rate_decimator[0] = 1;
cfg->ts_periodicity = layer_ids_length_;
} break;
case 2: {
static const unsigned int layer_ids[] = {0u, 1u};
layer_ids_ = layer_ids;
layer_ids_length_ = sizeof(layer_ids) / sizeof(*layer_ids);
static const int encode_flags[] = {
kTemporalUpdateLastAndGoldenRefAltRef,
kTemporalUpdateGoldenWithoutDependencyRefAltRef,
kTemporalUpdateLastRefAltRef, kTemporalUpdateGoldenRefAltRef,
kTemporalUpdateLastRefAltRef, kTemporalUpdateGoldenRefAltRef,
kTemporalUpdateLastRefAltRef, kTemporalUpdateNone
};
encode_flags_length_ = sizeof(encode_flags) / sizeof(*layer_ids);
encode_flags_ = encode_flags;
cfg->ts_rate_decimator[0] = 2;
cfg->ts_rate_decimator[1] = 1;
cfg->ts_periodicity = layer_ids_length_;
} break;
case 3: {
static const unsigned int layer_ids[] = {0u, 2u, 1u, 2u};
layer_ids_ = layer_ids;
layer_ids_length_ = sizeof(layer_ids) / sizeof(*layer_ids);
static const int encode_flags[] = {
kTemporalUpdateLastAndGoldenRefAltRef,
kTemporalUpdateNoneNoRefGoldenRefAltRef,
kTemporalUpdateGoldenWithoutDependencyRefAltRef, kTemporalUpdateNone,
kTemporalUpdateLastRefAltRef, kTemporalUpdateNone,
kTemporalUpdateGoldenRefAltRef, kTemporalUpdateNone
};
encode_flags_length_ = sizeof(encode_flags) / sizeof(*layer_ids);
encode_flags_ = encode_flags;
cfg->ts_rate_decimator[0] = 4;
cfg->ts_rate_decimator[1] = 2;
cfg->ts_rate_decimator[2] = 1;
cfg->ts_periodicity = layer_ids_length_;
} break;
default:
assert(false);
return false;
}
memcpy(
cfg->ts_layer_id, layer_ids_, sizeof(unsigned int) * layer_ids_length_);
return true;
}
virtual int EncodeFlags(uint32_t timestamp) {
frame_counter_++;
return CurrentEncodeFlags();
}
int CurrentEncodeFlags() const {
assert(encode_flags_length_ > 0 && encode_flags_ != NULL);
int index = frame_counter_ % encode_flags_length_;
assert(index >= 0 && index < encode_flags_length_);
return encode_flags_[index];
}
unsigned int CurrentLayerId() const {
assert(layer_ids_length_ > 0 && layer_ids_ != NULL);
int index = frame_counter_ % layer_ids_length_;
assert(index >= 0 && index < layer_ids_length_);
return layer_ids_[index];
}
virtual void PopulateCodecSpecific(bool base_layer_sync,
CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) {
assert(temporal_layers_ > 0);
if (temporal_layers_ == 1) {
vp8_info->temporalIdx = kNoTemporalIdx;
vp8_info->layerSync = false;
vp8_info->tl0PicIdx = kNoTl0PicIdx;
} else {
if (base_layer_sync) {
vp8_info->temporalIdx = 0;
vp8_info->layerSync = true;
} else {
vp8_info->temporalIdx = CurrentLayerId();
int temporal_reference = CurrentEncodeFlags();
if (temporal_reference == kTemporalUpdateAltrefWithoutDependency ||
temporal_reference == kTemporalUpdateGoldenWithoutDependency ||
temporal_reference ==
kTemporalUpdateGoldenWithoutDependencyRefAltRef ||
temporal_reference == kTemporalUpdateNoneNoRefGoldenRefAltRef ||
(temporal_reference == kTemporalUpdateNone &&
temporal_layers_ == 4)) {
vp8_info->layerSync = true;
} else {
vp8_info->layerSync = false;
}
}
if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) {
// Regardless of pattern the frame after a base layer sync will always
// be a layer sync.
vp8_info->layerSync = true;
}
if (vp8_info->temporalIdx == 0 && timestamp != timestamp_) {
timestamp_ = timestamp;
tl0_pic_idx_++;
}
last_base_layer_sync_ = base_layer_sync;
vp8_info->tl0PicIdx = tl0_pic_idx_;
}
}
void FrameEncoded(unsigned int size, uint32_t timestamp) {}
private:
int temporal_layers_;
int max_temporal_layers_;
int tl0_pic_idx_;
unsigned int frame_counter_;
uint32_t timestamp_;
bool last_base_layer_sync_;
// Pattern of temporal layer ids.
int layer_ids_length_;
const unsigned int* layer_ids_;
// Pattern of encode flags.
int encode_flags_length_;
const int* encode_flags_;
};
} // namespace
TemporalLayers* RealTimeTemporalLayersFactory::Create(
int max_temporal_layers,
uint8_t initial_tl0_pic_idx) const {
return new RealTimeTemporalLayers(max_temporal_layers, initial_tl0_pic_idx);
}
} // namespace webrtc

11
webrtc/modules/video_coding/codecs/vp8/temporal_layers.h
View File

@ -24,6 +24,8 @@ struct CodecSpecificInfoVP8;
class TemporalLayers {
public:
// Factory for TemporalLayer strategy. Default behaviour is a fixed pattern
// of temporal layers. See default_temporal_layers.cc
struct Factory {
Factory() {}
virtual ~Factory() {}
@ -49,5 +51,14 @@ class TemporalLayers {
virtual void FrameEncoded(unsigned int size, uint32_t timestamp) = 0;
};
// Factory for a temporal layers strategy that adaptively changes the number of
// layers based on input framerate so that the base layer has an acceptable
// framerate. See realtime_temporal_layers.cc
struct RealTimeTemporalLayersFactory : TemporalLayers::Factory {
virtual ~RealTimeTemporalLayersFactory() {}
virtual TemporalLayers* Create(int num_temporal_layers,
uint8_t initial_tl0_pic_idx) const;
};
} // namespace webrtc
#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_

1
webrtc/modules/video_coding/codecs/vp8/vp8.gyp
View File

@ -34,6 +34,7 @@
'vp8_impl.cc',
'default_temporal_layers.cc',
'default_temporal_layers.h',
'realtime_temporal_layers.cc',
'temporal_layers.h',
],
# Disable warnings to enable Win64 build, issue 1323.

140
webrtc/modules/video_coding/main/source/video_sender_unittest.cc
View File

@ -11,9 +11,11 @@
#include <vector>
#include "testing/gtest/include/gtest/gtest.h"
#include "webrtc/common.h"
#include "webrtc/common_video/test/frame_generator.h"
#include "webrtc/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h"
#include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h"
#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
#include "webrtc/modules/video_coding/main/interface/mock/mock_vcm_callbacks.h"
#include "webrtc/modules/video_coding/main/interface/video_coding.h"
#include "webrtc/modules/video_coding/main/source/video_coding_impl.h"
@ -31,12 +33,40 @@ using ::testing::Field;
using ::testing::NiceMock;
using ::testing::Pointee;
using ::testing::Return;
using ::testing::FloatEq;
using std::vector;
using webrtc::test::FrameGenerator;
namespace webrtc {
namespace vcm {
namespace {
enum {
kMaxNumberOfTemporalLayers = 3
};
struct Vp8StreamInfo {
float framerate_fps[kMaxNumberOfTemporalLayers];
int bitrate_kbps[kMaxNumberOfTemporalLayers];
};
MATCHER_P(MatchesVp8StreamInfo, expected, "") {
bool res = true;
for (int tl = 0; tl < kMaxNumberOfTemporalLayers; ++tl) {
if (abs(expected.framerate_fps[tl] - arg.framerate_fps[tl]) > 0.5) {
*result_listener << " framerate_fps[" << tl
<< "] = " << arg.framerate_fps[tl] << " (expected "
<< expected.framerate_fps[tl] << ") ";
res = false;
}
if (abs(expected.bitrate_kbps[tl] - arg.bitrate_kbps[tl]) > 10) {
*result_listener << " bitrate_kbps[" << tl
<< "] = " << arg.bitrate_kbps[tl] << " (expected "
<< expected.bitrate_kbps[tl] << ") ";
res = false;
}
}
return res;
}
class EmptyFrameGenerator : public FrameGenerator {
public:
@ -81,6 +111,15 @@ class PacketizationCallback : public VCMPacketizationCallback {
interval_ms();
}
Vp8StreamInfo CalculateVp8StreamInfo() {
Vp8StreamInfo info;
for (int tl = 0; tl < 3; ++tl) {
info.framerate_fps[tl] = FramerateFpsWithinTemporalLayer(tl);
info.bitrate_kbps[tl] = BitrateKbpsWithinTemporalLayer(tl);
}
return info;
}
private:
struct FrameData {
FrameData() {}
@ -319,6 +358,18 @@ class TestVideoSenderWithVp8 : public TestVideoSender {
}
}
Vp8StreamInfo SimulateWithFramerate(float framerate) {
const float short_simulation_interval = 5.0;
const float long_simulation_interval = 10.0;
// It appears that this 5 seconds simulation is needed to allow
// bitrate and framerate to stabilize.
InsertFrames(framerate, short_simulation_interval);
packetization_callback_.Reset();
InsertFrames(framerate, long_simulation_interval);
return packetization_callback_.CalculateVp8StreamInfo();
}
protected:
VideoCodec codec_;
int codec_bitrate_kbps_;
@ -327,51 +378,56 @@ class TestVideoSenderWithVp8 : public TestVideoSender {
TEST_F(TestVideoSenderWithVp8,
DISABLED_ON_ANDROID(FixedTemporalLayersStrategy)) {
// It appears that this 5 seconds simulation are need to allow
// bitrate and framerate to stabilize.
// TODO(andresp): the framerate calculation should be improved.
double framerate = 30.0;
InsertFrames(framerate, 5.0);
packetization_callback_.Reset();
const int low_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][0];
const int mid_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][1];
const int high_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][2];
{
Vp8StreamInfo expected = {{7.5, 15.0, 30.0}, {low_b, mid_b, high_b}};
EXPECT_THAT(SimulateWithFramerate(30.0), MatchesVp8StreamInfo(expected));
}
{
Vp8StreamInfo expected = {{3.75, 7.5, 15.0}, {low_b, mid_b, high_b}};
EXPECT_THAT(SimulateWithFramerate(15.0), MatchesVp8StreamInfo(expected));
}
}
// Need to simulate for 10 seconds due to VP8 bitrate controller.
InsertFrames(framerate, 10.0);
EXPECT_NEAR(
packetization_callback_.FramerateFpsWithinTemporalLayer(2), 30.0, 0.5);
EXPECT_NEAR(
packetization_callback_.FramerateFpsWithinTemporalLayer(1), 15.0, 0.5);
EXPECT_NEAR(
packetization_callback_.FramerateFpsWithinTemporalLayer(0), 7.5, 0.5);
EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(2),
codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][2],
10);
EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(1),
codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][1],
10);
EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(0),
codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][0],
10);
TEST_F(TestVideoSenderWithVp8,
DISABLED_ON_ANDROID(RealTimeTemporalLayersStrategy)) {
Config extra_options;
extra_options.Set<TemporalLayers::Factory>(
new RealTimeTemporalLayersFactory());
VideoCodec codec = MakeVp8VideoCodec(352, 288, 3);
codec.extra_options = &extra_options;
codec.minBitrate = 10;
codec.startBitrate = codec_bitrate_kbps_;
codec.maxBitrate = codec_bitrate_kbps_;
EXPECT_EQ(0, sender_->RegisterSendCodec(&codec, 1, 1200));
framerate = 15.0;
InsertFrames(framerate, 5.0);
packetization_callback_.Reset();
const int low_b = codec_bitrate_kbps_ * 0.4;
const int mid_b = codec_bitrate_kbps_ * 0.6;
const int high_b = codec_bitrate_kbps_;
InsertFrames(15.0, 10.0);
EXPECT_NEAR(
packetization_callback_.FramerateFpsWithinTemporalLayer(2), 15.0, 0.5);
EXPECT_NEAR(
packetization_callback_.FramerateFpsWithinTemporalLayer(1), 7.5, 0.5);
EXPECT_NEAR(
packetization_callback_.FramerateFpsWithinTemporalLayer(0), 3.75, 0.5);
EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(2),
codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][2],
10);
EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(1),
codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][1],
10);
EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(0),
codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][0],
10);
{
Vp8StreamInfo expected = {{7.5, 15.0, 30.0}, {low_b, mid_b, high_b}};
EXPECT_THAT(SimulateWithFramerate(30.0), MatchesVp8StreamInfo(expected));
}
{
Vp8StreamInfo expected = {{5.0, 10.0, 20.0}, {low_b, mid_b, high_b}};
EXPECT_THAT(SimulateWithFramerate(20.0), MatchesVp8StreamInfo(expected));
}
{
Vp8StreamInfo expected = {{7.5, 15.0, 15.0}, {mid_b, high_b, high_b}};
EXPECT_THAT(SimulateWithFramerate(15.0), MatchesVp8StreamInfo(expected));
}
{
Vp8StreamInfo expected = {{5.0, 10.0, 10.0}, {mid_b, high_b, high_b}};
EXPECT_THAT(SimulateWithFramerate(10.0), MatchesVp8StreamInfo(expected));
}
{
// TODO(andresp): Find out why this fails with framerate = 7.5
Vp8StreamInfo expected = {{7.0, 7.0, 7.0}, {high_b, high_b, high_b}};
EXPECT_THAT(SimulateWithFramerate(7.0), MatchesVp8StreamInfo(expected));
}
}
} // namespace
} // namespace vcm