Adding temporal layer strategy that keeps base layer framerate at an acceptable value.

R=stefan@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/2272005

git-svn-id: http://webrtc.googlecode.com/svn/trunk@4911 4adac7df-926f-26a2-2b94-8c16560cd09d

webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h

@@ -18,11 +18,11 @@ namespace webrtc {
 // Ratio allocation between temporal streams:
 // Values as required for the VP8 codec (accumulating).
 static const float
-  kVp8LayerRateAlloction[kMaxTemporalStreams][kMaxTemporalStreams] = {
+    kVp8LayerRateAlloction[kMaxTemporalStreams][kMaxTemporalStreams] = {
-    {1.0f, 0, 0, 0},  // 1 layer
+      {1.0f, 1.0f, 1.0f, 1.0f},  // 1 layer
-    {0.6f, 1.0f , 0 , 0},  // 2 layers {60%, 40%}
+      {0.6f, 1.0f, 1.0f, 1.0f},  // 2 layers {60%, 40%}
-    {0.4f, 0.6f , 1.0f, 0},  // 3 layers {40%, 20%, 40%}
+      {0.4f, 0.6f, 1.0f, 1.0f},  // 3 layers {40%, 20%, 40%}
-    {0.25f, 0.4f, 0.6f, 1.0f}  // 4 layers {25%, 15%, 20%, 40%}
+      {0.25f, 0.4f, 0.6f, 1.0f}  // 4 layers {25%, 15%, 20%, 40%}
 };
 
 }  // namespace webrtc

webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc

@@ -0,0 +1,268 @@
+/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+*
+*  Use of this source code is governed by a BSD-style license
+*  that can be found in the LICENSE file in the root of the source
+*  tree. An additional intellectual property rights grant can be found
+*  in the file PATENTS.  All contributing project authors may
+*  be found in the AUTHORS file in the root of the source tree.
+*/
+
+#include <stdlib.h>
+#include <algorithm>
+
+#include "vpx/vpx_encoder.h"
+#include "vpx/vp8cx.h"
+#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
+#include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h"
+#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
+
+// This file implements logic to adapt the number of temporal layers based on
+// input frame rate in order to avoid having the base layer being relaying at
+// a below acceptable framerate.
+namespace webrtc {
+namespace {
+enum {
+  kTemporalUpdateLast = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
+                        VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF,
+
+  kTemporalUpdateGolden =
+      VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
+
+  kTemporalUpdateGoldenWithoutDependency =
+      kTemporalUpdateGolden | VP8_EFLAG_NO_REF_GF,
+
+  kTemporalUpdateAltref = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_LAST,
+
+  kTemporalUpdateAltrefWithoutDependency =
+      kTemporalUpdateAltref | VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF,
+
+  kTemporalUpdateNone = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
+                        VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
+
+  kTemporalUpdateNoneNoRefAltref = kTemporalUpdateNone | VP8_EFLAG_NO_REF_ARF,
+
+  kTemporalUpdateNoneNoRefGoldenRefAltRef =
+      VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
+      VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
+
+  kTemporalUpdateGoldenWithoutDependencyRefAltRef =
+      VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
+
+  kTemporalUpdateLastRefAltRef =
+      VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
+
+  kTemporalUpdateGoldenRefAltRef = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
+
+  kTemporalUpdateLastAndGoldenRefAltRef =
+      VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
+
+  kTemporalUpdateLastRefAll = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_GF,
+};
+
+int CalculateNumberOfTemporalLayers(int current_temporal_layers,
+                                    int input_fps) {
+  if (input_fps >= 24) {
+    return 3;
+  }
+  if (input_fps >= 20 && current_temporal_layers >= 3) {
+    // Keep doing 3 temporal layers until we go below 20fps.
+    return 3;
+  }
+  if (input_fps >= 10) {
+    return 2;
+  }
+  if (input_fps > 8 && current_temporal_layers >= 2) {
+    // keep doing 2 temporal layers until we go below 8fps
+    return 2;
+  }
+  return 1;
+}
+
+class RealTimeTemporalLayers : public TemporalLayers {
+ public:
+  RealTimeTemporalLayers(int max_num_temporal_layers,
+                         uint8_t initial_tl0_pic_idx)
+      : temporal_layers_(1),
+        max_temporal_layers_(max_num_temporal_layers),
+        tl0_pic_idx_(initial_tl0_pic_idx),
+        frame_counter_(static_cast<unsigned int>(-1)),
+        timestamp_(0),
+        last_base_layer_sync_(0),
+        layer_ids_length_(0),
+        layer_ids_(NULL),
+        encode_flags_length_(0),
+        encode_flags_(NULL) {
+    assert(max_temporal_layers_ >= 1);
+    assert(max_temporal_layers_ <= 3);
+  }
+
+  virtual ~RealTimeTemporalLayers() {}
+
+  virtual bool ConfigureBitrates(int bitrate_kbit,
+                                 int max_bitrate_kbit,
+                                 int framerate,
+                                 vpx_codec_enc_cfg_t* cfg) {
+    temporal_layers_ =
+        CalculateNumberOfTemporalLayers(temporal_layers_, framerate);
+    temporal_layers_ = std::min(temporal_layers_, max_temporal_layers_);
+    assert(temporal_layers_ >= 1 && temporal_layers_ <= 3);
+
+    cfg->ts_number_layers = temporal_layers_;
+    for (int tl = 0; tl < temporal_layers_; ++tl) {
+      cfg->ts_target_bitrate[tl] =
+          bitrate_kbit * kVp8LayerRateAlloction[temporal_layers_ - 1][tl];
+    }
+
+    switch (temporal_layers_) {
+      case 1: {
+        static const unsigned int layer_ids[] = {0u};
+        layer_ids_ = layer_ids;
+        layer_ids_length_ = sizeof(layer_ids) / sizeof(*layer_ids);
+
+        static const int encode_flags[] = {kTemporalUpdateLastRefAll};
+        encode_flags_length_ = sizeof(encode_flags) / sizeof(*layer_ids);
+        encode_flags_ = encode_flags;
+
+        cfg->ts_rate_decimator[0] = 1;
+        cfg->ts_periodicity = layer_ids_length_;
+      } break;
+
+      case 2: {
+        static const unsigned int layer_ids[] = {0u, 1u};
+        layer_ids_ = layer_ids;
+        layer_ids_length_ = sizeof(layer_ids) / sizeof(*layer_ids);
+
+        static const int encode_flags[] = {
+          kTemporalUpdateLastAndGoldenRefAltRef,
+          kTemporalUpdateGoldenWithoutDependencyRefAltRef,
+          kTemporalUpdateLastRefAltRef, kTemporalUpdateGoldenRefAltRef,
+          kTemporalUpdateLastRefAltRef, kTemporalUpdateGoldenRefAltRef,
+          kTemporalUpdateLastRefAltRef, kTemporalUpdateNone
+        };
+        encode_flags_length_ = sizeof(encode_flags) / sizeof(*layer_ids);
+        encode_flags_ = encode_flags;
+
+        cfg->ts_rate_decimator[0] = 2;
+        cfg->ts_rate_decimator[1] = 1;
+        cfg->ts_periodicity = layer_ids_length_;
+      } break;
+
+      case 3: {
+        static const unsigned int layer_ids[] = {0u, 2u, 1u, 2u};
+        layer_ids_ = layer_ids;
+        layer_ids_length_ = sizeof(layer_ids) / sizeof(*layer_ids);
+
+        static const int encode_flags[] = {
+          kTemporalUpdateLastAndGoldenRefAltRef,
+          kTemporalUpdateNoneNoRefGoldenRefAltRef,
+          kTemporalUpdateGoldenWithoutDependencyRefAltRef, kTemporalUpdateNone,
+          kTemporalUpdateLastRefAltRef, kTemporalUpdateNone,
+          kTemporalUpdateGoldenRefAltRef, kTemporalUpdateNone
+        };
+        encode_flags_length_ = sizeof(encode_flags) / sizeof(*layer_ids);
+        encode_flags_ = encode_flags;
+
+        cfg->ts_rate_decimator[0] = 4;
+        cfg->ts_rate_decimator[1] = 2;
+        cfg->ts_rate_decimator[2] = 1;
+        cfg->ts_periodicity = layer_ids_length_;
+      } break;
+
+      default:
+        assert(false);
+        return false;
+    }
+    memcpy(
+        cfg->ts_layer_id, layer_ids_, sizeof(unsigned int) * layer_ids_length_);
+    return true;
+  }
+
+  virtual int EncodeFlags(uint32_t timestamp) {
+    frame_counter_++;
+    return CurrentEncodeFlags();
+  }
+
+  int CurrentEncodeFlags() const {
+    assert(encode_flags_length_ > 0 && encode_flags_ != NULL);
+    int index = frame_counter_ % encode_flags_length_;
+    assert(index >= 0 && index < encode_flags_length_);
+    return encode_flags_[index];
+  }
+
+  unsigned int CurrentLayerId() const {
+    assert(layer_ids_length_ > 0 && layer_ids_ != NULL);
+    int index = frame_counter_ % layer_ids_length_;
+    assert(index >= 0 && index < layer_ids_length_);
+    return layer_ids_[index];
+  }
+
+  virtual void PopulateCodecSpecific(bool base_layer_sync,
+                                     CodecSpecificInfoVP8* vp8_info,
+                                     uint32_t timestamp) {
+    assert(temporal_layers_ > 0);
+
+    if (temporal_layers_ == 1) {
+      vp8_info->temporalIdx = kNoTemporalIdx;
+      vp8_info->layerSync = false;
+      vp8_info->tl0PicIdx = kNoTl0PicIdx;
+    } else {
+      if (base_layer_sync) {
+        vp8_info->temporalIdx = 0;
+        vp8_info->layerSync = true;
+      } else {
+        vp8_info->temporalIdx = CurrentLayerId();
+        int temporal_reference = CurrentEncodeFlags();
+
+        if (temporal_reference == kTemporalUpdateAltrefWithoutDependency ||
+            temporal_reference == kTemporalUpdateGoldenWithoutDependency ||
+            temporal_reference ==
+                kTemporalUpdateGoldenWithoutDependencyRefAltRef ||
+            temporal_reference == kTemporalUpdateNoneNoRefGoldenRefAltRef ||
+            (temporal_reference == kTemporalUpdateNone &&
+             temporal_layers_ == 4)) {
+          vp8_info->layerSync = true;
+        } else {
+          vp8_info->layerSync = false;
+        }
+      }
+      if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) {
+        // Regardless of pattern the frame after a base layer sync will always
+        // be a layer sync.
+        vp8_info->layerSync = true;
+      }
+      if (vp8_info->temporalIdx == 0 && timestamp != timestamp_) {
+        timestamp_ = timestamp;
+        tl0_pic_idx_++;
+      }
+      last_base_layer_sync_ = base_layer_sync;
+      vp8_info->tl0PicIdx = tl0_pic_idx_;
+    }
+  }
+
+  void FrameEncoded(unsigned int size, uint32_t timestamp) {}
+
+ private:
+  int temporal_layers_;
+  int max_temporal_layers_;
+
+  int tl0_pic_idx_;
+  unsigned int frame_counter_;
+  uint32_t timestamp_;
+  bool last_base_layer_sync_;
+
+  // Pattern of temporal layer ids.
+  int layer_ids_length_;
+  const unsigned int* layer_ids_;
+
+  // Pattern of encode flags.
+  int encode_flags_length_;
+  const int* encode_flags_;
+};
+}  // namespace
+
+TemporalLayers* RealTimeTemporalLayersFactory::Create(
+    int max_temporal_layers,
+    uint8_t initial_tl0_pic_idx) const {
+  return new RealTimeTemporalLayers(max_temporal_layers, initial_tl0_pic_idx);
+}
+}  // namespace webrtc

webrtc/modules/video_coding/codecs/vp8/temporal_layers.h

@@ -24,6 +24,8 @@ struct CodecSpecificInfoVP8;
 
 class TemporalLayers {
  public:
+  // Factory for TemporalLayer strategy. Default behaviour is a fixed pattern
+  // of temporal layers. See default_temporal_layers.cc
   struct Factory {
     Factory() {}
     virtual ~Factory() {}
@@ -49,5 +51,14 @@ class TemporalLayers {
   virtual void FrameEncoded(unsigned int size, uint32_t timestamp) = 0;
 };
 
+// Factory for a temporal layers strategy that adaptively changes the number of
+// layers based on input framerate so that the base layer has an acceptable
+// framerate. See realtime_temporal_layers.cc
+struct RealTimeTemporalLayersFactory : TemporalLayers::Factory {
+  virtual ~RealTimeTemporalLayersFactory() {}
+  virtual TemporalLayers* Create(int num_temporal_layers,
+                                 uint8_t initial_tl0_pic_idx) const;
+};
+
 }  // namespace webrtc
 #endif  // WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_

webrtc/modules/video_coding/codecs/vp8/vp8.gyp

@@ -34,6 +34,7 @@
         'vp8_impl.cc',
         'default_temporal_layers.cc',
         'default_temporal_layers.h',
+        'realtime_temporal_layers.cc',
         'temporal_layers.h',
       ],
       # Disable warnings to enable Win64 build, issue 1323.

webrtc/modules/video_coding/main/source/video_sender_unittest.cc

@@ -11,9 +11,11 @@
 #include <vector>
 
 #include "testing/gtest/include/gtest/gtest.h"
+#include "webrtc/common.h"
 #include "webrtc/common_video/test/frame_generator.h"
 #include "webrtc/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h"
 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h"
+#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
 #include "webrtc/modules/video_coding/main/interface/mock/mock_vcm_callbacks.h"
 #include "webrtc/modules/video_coding/main/interface/video_coding.h"
 #include "webrtc/modules/video_coding/main/source/video_coding_impl.h"
@@ -31,12 +33,40 @@ using ::testing::Field;
 using ::testing::NiceMock;
 using ::testing::Pointee;
 using ::testing::Return;
+using ::testing::FloatEq;
 using std::vector;
 using webrtc::test::FrameGenerator;
 
 namespace webrtc {
 namespace vcm {
 namespace {
+enum {
+  kMaxNumberOfTemporalLayers = 3
+};
+
+struct Vp8StreamInfo {
+  float framerate_fps[kMaxNumberOfTemporalLayers];
+  int bitrate_kbps[kMaxNumberOfTemporalLayers];
+};
+
+MATCHER_P(MatchesVp8StreamInfo, expected, "") {
+  bool res = true;
+  for (int tl = 0; tl < kMaxNumberOfTemporalLayers; ++tl) {
+    if (abs(expected.framerate_fps[tl] - arg.framerate_fps[tl]) > 0.5) {
+      *result_listener << " framerate_fps[" << tl
+                       << "] = " << arg.framerate_fps[tl] << " (expected "
+                       << expected.framerate_fps[tl] << ") ";
+      res = false;
+    }
+    if (abs(expected.bitrate_kbps[tl] - arg.bitrate_kbps[tl]) > 10) {
+      *result_listener << " bitrate_kbps[" << tl
+                       << "] = " << arg.bitrate_kbps[tl] << " (expected "
+                       << expected.bitrate_kbps[tl] << ") ";
+      res = false;
+    }
+  }
+  return res;
+}
 
 class EmptyFrameGenerator : public FrameGenerator {
  public:
@@ -81,6 +111,15 @@ class PacketizationCallback : public VCMPacketizationCallback {
            interval_ms();
   }
 
+  Vp8StreamInfo CalculateVp8StreamInfo() {
+    Vp8StreamInfo info;
+    for (int tl = 0; tl < 3; ++tl) {
+      info.framerate_fps[tl] = FramerateFpsWithinTemporalLayer(tl);
+      info.bitrate_kbps[tl] = BitrateKbpsWithinTemporalLayer(tl);
+    }
+    return info;
+  }
+
  private:
   struct FrameData {
     FrameData() {}
@@ -319,6 +358,18 @@ class TestVideoSenderWithVp8 : public TestVideoSender {
     }
   }
 
+  Vp8StreamInfo SimulateWithFramerate(float framerate) {
+    const float short_simulation_interval = 5.0;
+    const float long_simulation_interval = 10.0;
+    // It appears that this 5 seconds simulation is needed to allow
+    // bitrate and framerate to stabilize.
+    InsertFrames(framerate, short_simulation_interval);
+    packetization_callback_.Reset();
+
+    InsertFrames(framerate, long_simulation_interval);
+    return packetization_callback_.CalculateVp8StreamInfo();
+  }
+
  protected:
   VideoCodec codec_;
   int codec_bitrate_kbps_;
@@ -327,51 +378,56 @@ class TestVideoSenderWithVp8 : public TestVideoSender {
 
 TEST_F(TestVideoSenderWithVp8,
        DISABLED_ON_ANDROID(FixedTemporalLayersStrategy)) {
-  // It appears that this 5 seconds simulation are need to allow
+  const int low_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][0];
-  // bitrate and framerate to stabilize.
+  const int mid_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][1];
-  // TODO(andresp): the framerate calculation should be improved.
+  const int high_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][2];
-  double framerate = 30.0;
+  {
-  InsertFrames(framerate, 5.0);
+    Vp8StreamInfo expected = {{7.5, 15.0, 30.0}, {low_b, mid_b, high_b}};
-  packetization_callback_.Reset();
+    EXPECT_THAT(SimulateWithFramerate(30.0), MatchesVp8StreamInfo(expected));
+  }
+  {
+    Vp8StreamInfo expected = {{3.75, 7.5, 15.0}, {low_b, mid_b, high_b}};
+    EXPECT_THAT(SimulateWithFramerate(15.0), MatchesVp8StreamInfo(expected));
+  }
+}
 
-  // Need to simulate for 10 seconds due to VP8 bitrate controller.
+TEST_F(TestVideoSenderWithVp8,
-  InsertFrames(framerate, 10.0);
+       DISABLED_ON_ANDROID(RealTimeTemporalLayersStrategy)) {
-  EXPECT_NEAR(
+  Config extra_options;
-      packetization_callback_.FramerateFpsWithinTemporalLayer(2), 30.0, 0.5);
+  extra_options.Set<TemporalLayers::Factory>(
-  EXPECT_NEAR(
+      new RealTimeTemporalLayersFactory());
-      packetization_callback_.FramerateFpsWithinTemporalLayer(1), 15.0, 0.5);
+  VideoCodec codec = MakeVp8VideoCodec(352, 288, 3);
-  EXPECT_NEAR(
+  codec.extra_options = &extra_options;
-      packetization_callback_.FramerateFpsWithinTemporalLayer(0), 7.5, 0.5);
+  codec.minBitrate = 10;
-  EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(2),
+  codec.startBitrate = codec_bitrate_kbps_;
-              codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][2],
+  codec.maxBitrate = codec_bitrate_kbps_;
-              10);
+  EXPECT_EQ(0, sender_->RegisterSendCodec(&codec, 1, 1200));
-  EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(1),
-              codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][1],
-              10);
-  EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(0),
-              codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][0],
-              10);
 
-  framerate = 15.0;
+  const int low_b = codec_bitrate_kbps_ * 0.4;
-  InsertFrames(framerate, 5.0);
+  const int mid_b = codec_bitrate_kbps_ * 0.6;
-  packetization_callback_.Reset();
+  const int high_b = codec_bitrate_kbps_;
 
-  InsertFrames(15.0, 10.0);
+  {
-  EXPECT_NEAR(
+    Vp8StreamInfo expected = {{7.5, 15.0, 30.0}, {low_b, mid_b, high_b}};
-      packetization_callback_.FramerateFpsWithinTemporalLayer(2), 15.0, 0.5);
+    EXPECT_THAT(SimulateWithFramerate(30.0), MatchesVp8StreamInfo(expected));
-  EXPECT_NEAR(
+  }
-      packetization_callback_.FramerateFpsWithinTemporalLayer(1), 7.5, 0.5);
+  {
-  EXPECT_NEAR(
+    Vp8StreamInfo expected = {{5.0, 10.0, 20.0}, {low_b, mid_b, high_b}};
-      packetization_callback_.FramerateFpsWithinTemporalLayer(0), 3.75, 0.5);
+    EXPECT_THAT(SimulateWithFramerate(20.0), MatchesVp8StreamInfo(expected));
-  EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(2),
+  }
-              codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][2],
+  {
-              10);
+    Vp8StreamInfo expected = {{7.5, 15.0, 15.0}, {mid_b, high_b, high_b}};
-  EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(1),
+    EXPECT_THAT(SimulateWithFramerate(15.0), MatchesVp8StreamInfo(expected));
-              codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][1],
+  }
-              10);
+  {
-  EXPECT_NEAR(packetization_callback_.BitrateKbpsWithinTemporalLayer(0),
+    Vp8StreamInfo expected = {{5.0, 10.0, 10.0}, {mid_b, high_b, high_b}};
-              codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][0],
+    EXPECT_THAT(SimulateWithFramerate(10.0), MatchesVp8StreamInfo(expected));
-              10);
+  }
+  {
+    // TODO(andresp): Find out why this fails with framerate = 7.5
+    Vp8StreamInfo expected = {{7.0, 7.0, 7.0}, {high_b, high_b, high_b}};
+    EXPECT_THAT(SimulateWithFramerate(7.0), MatchesVp8StreamInfo(expected));
+  }
 }
 }  // namespace
 }  // namespace vcm