Add HW fallback option to software encoding.

Permits falling back to software encoding for unsupported resolutions.

BUG=chromium:475116, chromium:487934
R=mflodman@webrtc.org, stefan@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/46279004

Cr-Commit-Position: refs/heads/master@{#9227}

talk/media/webrtc/webrtcvideoengine2.cc

@@ -1634,6 +1634,21 @@ WebRtcVideoChannel2::WebRtcVideoSendStream::VideoSendStreamParameters::
       codec_settings(codec_settings) {
 }
 
+WebRtcVideoChannel2::WebRtcVideoSendStream::AllocatedEncoder::AllocatedEncoder(
+    webrtc::VideoEncoder* encoder,
+    webrtc::VideoCodecType type,
+    bool external)
+    : encoder(encoder),
+      external_encoder(nullptr),
+      type(type),
+      external(external) {
+  if (external) {
+    external_encoder = encoder;
+    this->encoder =
+        new webrtc::VideoEncoderSoftwareFallbackWrapper(type, encoder);
+  }
+}
+
 WebRtcVideoChannel2::WebRtcVideoSendStream::WebRtcVideoSendStream(
     webrtc::Call* call,
     WebRtcVideoEncoderFactory* external_encoder_factory,
@@ -1885,10 +1900,9 @@ WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoder(
 void WebRtcVideoChannel2::WebRtcVideoSendStream::DestroyVideoEncoder(
     AllocatedEncoder* encoder) {
   if (encoder->external) {
-    external_encoder_factory_->DestroyVideoEncoder(encoder->encoder);
-  } else {
-    delete encoder->encoder;
+    external_encoder_factory_->DestroyVideoEncoder(encoder->external_encoder);
   }
+  delete encoder->encoder;
 }
 
 void WebRtcVideoChannel2::WebRtcVideoSendStream::SetCodecAndOptions(

talk/media/webrtc/webrtcvideoengine2.h

@@ -321,9 +321,9 @@ class WebRtcVideoChannel2 : public rtc::MessageHandler,
     struct AllocatedEncoder {
       AllocatedEncoder(webrtc::VideoEncoder* encoder,
                        webrtc::VideoCodecType type,
-                       bool external)
-          : encoder(encoder), type(type), external(external) {}
+                       bool external);
       webrtc::VideoEncoder* encoder;
+      webrtc::VideoEncoder* external_encoder;
       webrtc::VideoCodecType type;
       bool external;
     };

webrtc/video/BUILD.gn

@@ -22,6 +22,7 @@ source_set("video") {
     "transport_adapter.cc",
     "transport_adapter.h",
     "video_decoder.cc",
+    "video_encoder.cc",
     "video_receive_stream.cc",
     "video_receive_stream.h",
     "video_send_stream.cc",

webrtc/video/call.cc

@@ -36,16 +36,6 @@
 #include "webrtc/video/video_send_stream.h"
 
 namespace webrtc {
-VideoEncoder* VideoEncoder::Create(VideoEncoder::EncoderType codec_type) {
-  switch (codec_type) {
-    case kVp8:
-      return VP8Encoder::Create();
-    case kVp9:
-      return VP9Encoder::Create();
-  }
-  RTC_NOTREACHED();
-  return nullptr;
-}
 
 const int Call::Config::kDefaultStartBitrateBps = 300000;
 

webrtc/video/video_encoder.cc

@@ -0,0 +1,125 @@
+/*
+ *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/video_encoder.h"
+
+#include "webrtc/base/checks.h"
+#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
+#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
+#include "webrtc/system_wrappers/interface/logging.h"
+
+namespace webrtc {
+VideoEncoder* VideoEncoder::Create(VideoEncoder::EncoderType codec_type) {
+  switch (codec_type) {
+    case kVp8:
+      return VP8Encoder::Create();
+    case kVp9:
+      return VP9Encoder::Create();
+    case kUnsupportedCodec:
+      RTC_NOTREACHED();
+      return nullptr;
+  }
+  RTC_NOTREACHED();
+  return nullptr;
+}
+
+VideoEncoder::EncoderType CodecToEncoderType(VideoCodecType codec_type) {
+  switch (codec_type) {
+    case kVideoCodecVP8:
+      return VideoEncoder::kVp8;
+    case kVideoCodecVP9:
+      return VideoEncoder::kVp9;
+    default:
+      return VideoEncoder::kUnsupportedCodec;
+  }
+}
+
+VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper(
+    VideoCodecType codec_type,
+    webrtc::VideoEncoder* encoder)
+    : encoder_type_(CodecToEncoderType(codec_type)),
+      encoder_(encoder),
+      callback_(nullptr) {
+}
+
+int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode(
+    const VideoCodec* codec_settings,
+    int32_t number_of_cores,
+    size_t max_payload_size) {
+  int32_t ret =
+      encoder_->InitEncode(codec_settings, number_of_cores, max_payload_size);
+  if (ret == WEBRTC_VIDEO_CODEC_OK || encoder_type_ == kUnsupportedCodec) {
+    fallback_encoder_.reset();
+    if (callback_)
+      encoder_->RegisterEncodeCompleteCallback(callback_);
+    return ret;
+  }
+  // Try to instantiate software codec.
+  fallback_encoder_.reset(VideoEncoder::Create(encoder_type_));
+  if (fallback_encoder_->InitEncode(codec_settings, number_of_cores,
+                                    max_payload_size) ==
+      WEBRTC_VIDEO_CODEC_OK) {
+    if (callback_)
+      fallback_encoder_->RegisterEncodeCompleteCallback(callback_);
+    return WEBRTC_VIDEO_CODEC_OK;
+  }
+  // Software encoder failed, reset and use original return code.
+  fallback_encoder_.reset();
+  return ret;
+}
+
+int32_t VideoEncoderSoftwareFallbackWrapper::RegisterEncodeCompleteCallback(
+    EncodedImageCallback* callback) {
+  callback_ = callback;
+  int32_t ret = encoder_->RegisterEncodeCompleteCallback(callback);
+  if (fallback_encoder_)
+    return fallback_encoder_->RegisterEncodeCompleteCallback(callback);
+  return ret;
+}
+
+int32_t VideoEncoderSoftwareFallbackWrapper::Release() {
+  if (fallback_encoder_)
+    return fallback_encoder_->Release();
+  return encoder_->Release();
+}
+
+int32_t VideoEncoderSoftwareFallbackWrapper::Encode(
+    const I420VideoFrame& frame,
+    const CodecSpecificInfo* codec_specific_info,
+    const std::vector<VideoFrameType>* frame_types) {
+  if (fallback_encoder_)
+    return fallback_encoder_->Encode(frame, codec_specific_info, frame_types);
+  return encoder_->Encode(frame, codec_specific_info, frame_types);
+}
+
+int32_t VideoEncoderSoftwareFallbackWrapper::SetChannelParameters(
+    uint32_t packet_loss,
+    int64_t rtt) {
+  int32_t ret = encoder_->SetChannelParameters(packet_loss, rtt);
+  if (fallback_encoder_)
+    return fallback_encoder_->SetChannelParameters(packet_loss, rtt);
+  return ret;
+}
+
+int32_t VideoEncoderSoftwareFallbackWrapper::SetRates(uint32_t bitrate,
+                                                      uint32_t framerate) {
+  int32_t ret = encoder_->SetRates(bitrate, framerate);
+  if (fallback_encoder_)
+    return fallback_encoder_->SetRates(bitrate, framerate);
+  return ret;
+}
+
+void VideoEncoderSoftwareFallbackWrapper::OnDroppedFrame() {
+  if (fallback_encoder_)
+    return fallback_encoder_->OnDroppedFrame();
+  return encoder_->OnDroppedFrame();
+}
+
+}  // namespace webrtc

webrtc/video/video_encoder_unittest.cc

@@ -0,0 +1,200 @@
+/*
+ *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/video_encoder.h"
+
+#include "testing/gtest/include/gtest/gtest.h"
+#include "webrtc/modules/video_coding/codecs/interface/video_error_codes.h"
+
+namespace webrtc {
+
+const size_t kMaxPayloadSize = 800;
+
+class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test {
+ protected:
+  VideoEncoderSoftwareFallbackWrapperTest()
+      : fallback_wrapper_(kVideoCodecVP8, &fake_encoder_) {}
+
+  class CountingFakeEncoder : public VideoEncoder {
+   public:
+    int32_t InitEncode(const VideoCodec* codec_settings,
+                       int32_t number_of_cores,
+                       size_t max_payload_size) override {
+      ++init_encode_count_;
+      return init_encode_return_code_;
+    }
+    int32_t Encode(const I420VideoFrame& frame,
+                   const CodecSpecificInfo* codec_specific_info,
+                   const std::vector<VideoFrameType>* frame_types) override {
+      ++encode_count_;
+      return WEBRTC_VIDEO_CODEC_OK;
+    }
+
+    int32_t RegisterEncodeCompleteCallback(
+        EncodedImageCallback* callback) override {
+      encode_complete_callback_ = callback;
+      return WEBRTC_VIDEO_CODEC_OK;
+    }
+
+    int32_t Release() override {
+      ++release_count_;
+      return WEBRTC_VIDEO_CODEC_OK;
+    }
+
+    int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) override {
+      ++set_channel_parameters_count_;
+      return WEBRTC_VIDEO_CODEC_OK;
+    }
+
+    int32_t SetRates(uint32_t bitrate, uint32_t framerate) override {
+      ++set_rates_count_;
+      return WEBRTC_VIDEO_CODEC_OK;
+    }
+
+    void OnDroppedFrame() override { ++on_dropped_frame_count_; }
+
+    int init_encode_count_ = 0;
+    int32_t init_encode_return_code_ = WEBRTC_VIDEO_CODEC_OK;
+    int encode_count_ = 0;
+    EncodedImageCallback* encode_complete_callback_ = nullptr;
+    int release_count_ = 0;
+    int set_channel_parameters_count_ = 0;
+    int set_rates_count_ = 0;
+    int on_dropped_frame_count_ = 0;
+  };
+
+  class FakeEncodedImageCallback : public EncodedImageCallback {
+   public:
+    int32_t Encoded(const EncodedImage& encoded_image,
+                    const CodecSpecificInfo* codec_specific_info,
+                    const RTPFragmentationHeader* fragmentation) override {
+      return ++callback_count_;
+    }
+    int callback_count_ = 0;
+  };
+
+  void UtilizeFallbackEncoder();
+
+  FakeEncodedImageCallback callback_;
+  CountingFakeEncoder fake_encoder_;
+  VideoEncoderSoftwareFallbackWrapper fallback_wrapper_;
+  VideoCodec codec_ = {};
+  I420VideoFrame frame_;
+};
+
+void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() {
+  static const int kWidth = 320;
+  static const int kHeight = 240;
+  fallback_wrapper_.RegisterEncodeCompleteCallback(&callback_);
+  EXPECT_EQ(&callback_, fake_encoder_.encode_complete_callback_);
+
+  // Register with failing fake encoder. Should succeed with VP8 fallback.
+  codec_.codecType = kVideoCodecVP8;
+  codec_.maxFramerate = 30;
+  codec_.width = kWidth;
+  codec_.height = kHeight;
+  fake_encoder_.init_encode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR;
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+            fallback_wrapper_.InitEncode(&codec_, 2, kMaxPayloadSize));
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.SetRates(300, 30));
+
+  frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, (kWidth + 1) / 2,
+                          (kWidth + 1) / 2);
+  memset(frame_.buffer(webrtc::kYPlane), 16,
+         frame_.allocated_size(webrtc::kYPlane));
+  memset(frame_.buffer(webrtc::kUPlane), 128,
+         frame_.allocated_size(webrtc::kUPlane));
+  memset(frame_.buffer(webrtc::kVPlane), 128,
+         frame_.allocated_size(webrtc::kVPlane));
+
+  std::vector<VideoFrameType> types(1, kKeyFrame);
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+            fallback_wrapper_.Encode(frame_, nullptr, &types));
+  EXPECT_EQ(0, fake_encoder_.encode_count_);
+  EXPECT_GT(callback_.callback_count_, 0);
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest, InitializesEncoder) {
+  VideoCodec codec = {};
+  fallback_wrapper_.InitEncode(&codec, 2, kMaxPayloadSize);
+  EXPECT_EQ(1, fake_encoder_.init_encode_count_);
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest, CanUtilizeFallbackEncoder) {
+  UtilizeFallbackEncoder();
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
+       InternalEncoderNotReleasedDuringFallback) {
+  UtilizeFallbackEncoder();
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
+  EXPECT_EQ(0, fake_encoder_.release_count_);
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
+       InternalEncoderNotEncodingDuringFallback) {
+  UtilizeFallbackEncoder();
+  EXPECT_EQ(0, fake_encoder_.encode_count_);
+
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
+       CanRegisterCallbackWhileUsingFallbackEncoder) {
+  UtilizeFallbackEncoder();
+  // Registering an encode-complete callback should still work when fallback
+  // encoder is being used.
+  FakeEncodedImageCallback callback2;
+  fallback_wrapper_.RegisterEncodeCompleteCallback(&callback2);
+  EXPECT_EQ(&callback2, fake_encoder_.encode_complete_callback_);
+
+  // Encoding a frame using the fallback should arrive at the new callback.
+  std::vector<VideoFrameType> types(1, kKeyFrame);
+  frame_.set_timestamp(frame_.timestamp() + 1000);
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+            fallback_wrapper_.Encode(frame_, nullptr, &types));
+
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
+       SetChannelParametersForwardedDuringFallback) {
+  UtilizeFallbackEncoder();
+  EXPECT_EQ(0, fake_encoder_.set_channel_parameters_count_);
+  fallback_wrapper_.SetChannelParameters(1, 1);
+  EXPECT_EQ(1, fake_encoder_.set_channel_parameters_count_);
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
+       SetRatesForwardedDuringFallback) {
+  UtilizeFallbackEncoder();
+  EXPECT_EQ(1, fake_encoder_.set_rates_count_);
+  fallback_wrapper_.SetRates(1, 1);
+  EXPECT_EQ(2, fake_encoder_.set_rates_count_);
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
+       OnDroppedFrameForwardedWithoutFallback) {
+  fallback_wrapper_.OnDroppedFrame();
+  EXPECT_EQ(1, fake_encoder_.on_dropped_frame_count_);
+}
+
+TEST_F(VideoEncoderSoftwareFallbackWrapperTest,
+       OnDroppedFrameNotForwardedDuringFallback) {
+  UtilizeFallbackEncoder();
+  fallback_wrapper_.OnDroppedFrame();
+  EXPECT_EQ(0, fake_encoder_.on_dropped_frame_count_);
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release());
+}
+
+}  // namespace webrtc

webrtc/video/webrtc_video.gypi

@@ -23,6 +23,7 @@
       'video/transport_adapter.cc',
       'video/transport_adapter.h',
       'video/video_decoder.cc',
+      'video/video_encoder.cc',
       'video/video_receive_stream.cc',
       'video/video_receive_stream.h',
       'video/video_send_stream.cc',

webrtc/video_encoder.h

@@ -39,6 +39,7 @@ class VideoEncoder {
   enum EncoderType {
     kVp8,
     kVp9,
+    kUnsupportedCodec,
   };
 
   static VideoEncoder* Create(EncoderType codec_type);
@@ -125,5 +126,36 @@ class VideoEncoder {
   virtual void OnDroppedFrame() {};
 };
 
+// Class used to wrap external VideoEncoders to provide a fallback option on
+// software encoding when a hardware encoder fails to encode a stream due to
+// hardware restrictions, such as max resolution.
+class VideoEncoderSoftwareFallbackWrapper : public VideoEncoder {
+ public:
+  VideoEncoderSoftwareFallbackWrapper(VideoCodecType codec_type,
+                                      webrtc::VideoEncoder* encoder);
+
+  int32_t InitEncode(const VideoCodec* codec_settings,
+                     int32_t number_of_cores,
+                     size_t max_payload_size) override;
+
+  int32_t RegisterEncodeCompleteCallback(
+      EncodedImageCallback* callback) override;
+
+  int32_t Release() override;
+  int32_t Encode(const I420VideoFrame& frame,
+                 const CodecSpecificInfo* codec_specific_info,
+                 const std::vector<VideoFrameType>* frame_types) override;
+  int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) override;
+
+  int32_t SetRates(uint32_t bitrate, uint32_t framerate) override;
+  void OnDroppedFrame() override;
+
+ private:
+  const EncoderType encoder_type_;
+  webrtc::VideoEncoder* const encoder_;
+
+  rtc::scoped_ptr<webrtc::VideoEncoder> fallback_encoder_;
+  EncodedImageCallback* callback_;
+};
 }  // namespace webrtc
 #endif  // WEBRTC_VIDEO_ENCODER_H_

webrtc/webrtc_tests.gypi

@@ -151,6 +151,7 @@
         'video/end_to_end_tests.cc',
         'video/send_statistics_proxy_unittest.cc',
         'video/video_decoder_unittest.cc',
+        'video/video_encoder_unittest.cc',
         'video/video_send_stream_tests.cc',
       ],
       'dependencies': [