From 52cd828e1731266272e671020c353f5f89992a83 Mon Sep 17 00:00:00 2001 From: "pthatcher@webrtc.org" Date: Wed, 18 Mar 2015 02:24:43 +0000 Subject: [PATCH] Allow webrtc external encoder factories to declare encoders have internal camera sources. This flag is passed to existing VieExternalCodec API (and others) to denote encoders that don't require/expect frames from the normal capture pipeline. This is the simplest way to allow camera->encoder texture support, until textures are supported through the normal camera pipeline and the lifetime issues are all figured out (I hear this is on the backlog, but not there yet). Ideally, the flag would be on the encoder, but that doesn't work with SimulcastEncoderAdapter, since it doesn't create an encoder right away. Note that this change only affects WebRtcVideoEngine (not WRVE2), since WRVE2 uses video_send_stream, and my hope is that by the time things have switched to WRVE2, textures will be supported with the normal camera pipeline and the dependency on internal sources can be thrown away. BUG= R=pbos@webrtc.org, pthatcher@webrtc.org Review URL: https://webrtc-codereview.appspot.com/42349004 Cr-Commit-Position: refs/heads/master@{#8769} git-svn-id: http://webrtc.googlecode.com/svn/trunk@8769 4adac7df-926f-26a2-2b94-8c16560cd09d --- talk/media/webrtc/fakewebrtcvideoengine.h | 47 ++++++++++++++----- talk/media/webrtc/webrtcvideoencoderfactory.h | 9 ++++ talk/media/webrtc/webrtcvideoengine.cc | 15 ++++-- talk/media/webrtc/webrtcvideoengine.h | 8 +++- .../webrtc/webrtcvideoengine_unittest.cc | 24 ++++++++++ 5 files changed, 88 insertions(+), 15 deletions(-) diff --git a/talk/media/webrtc/fakewebrtcvideoengine.h b/talk/media/webrtc/fakewebrtcvideoengine.h index 16c7f8b71..eb9790bc8 100644 --- a/talk/media/webrtc/fakewebrtcvideoengine.h +++ b/talk/media/webrtc/fakewebrtcvideoengine.h @@ -206,8 +206,7 @@ class FakeWebRtcVideoEncoder : public webrtc::VideoEncoder { class FakeWebRtcVideoEncoderFactory : public WebRtcVideoEncoderFactory { public: FakeWebRtcVideoEncoderFactory() - : num_created_encoders_(0) { - } + : num_created_encoders_(0), encoders_have_internal_sources_(false) {} virtual webrtc::VideoEncoder* CreateVideoEncoder( webrtc::VideoCodecType type) { @@ -232,6 +231,15 @@ class FakeWebRtcVideoEncoderFactory : public WebRtcVideoEncoderFactory { return codecs_; } + virtual bool EncoderTypeHasInternalSource( + webrtc::VideoCodecType type) const override { + return encoders_have_internal_sources_; + } + + void set_encoders_have_internal_sources(bool internal_source) { + encoders_have_internal_sources_ = internal_source; + } + void AddSupportedVideoCodecType(webrtc::VideoCodecType type, const std::string& name) { supported_codec_types_.insert(type); @@ -252,6 +260,12 @@ class FakeWebRtcVideoEncoderFactory : public WebRtcVideoEncoderFactory { std::vector codecs_; std::vector encoders_; int num_created_encoders_; + bool encoders_have_internal_sources_; +}; + +// Information associated with an external encoder. +struct ExternalEncoderInfo { + bool internal_source; }; class FakeWebRtcVideoEngine @@ -336,7 +350,7 @@ class FakeWebRtcVideoEngine bool hybrid_nack_fec_; std::vector recv_codecs; std::set ext_decoder_pl_types_; - std::set ext_encoder_pl_types_; + std::map ext_encoders_; webrtc::VideoCodec send_codec; unsigned int send_video_bitrate_; unsigned int send_fec_bitrate_; @@ -603,20 +617,27 @@ class FakeWebRtcVideoEngine bool ExternalEncoderRegistered(int channel, unsigned int pl_type) const { WEBRTC_ASSERT_CHANNEL(channel); - return channels_.find(channel)->second-> - ext_encoder_pl_types_.count(pl_type) != 0; + return channels_.find(channel)->second->ext_encoders_.count(pl_type) != 0; }; int GetNumExternalEncoderRegistered(int channel) const { WEBRTC_ASSERT_CHANNEL(channel); return static_cast( - channels_.find(channel)->second->ext_encoder_pl_types_.size()); + channels_.find(channel)->second->ext_encoders_.size()); + }; + bool ExternalEncoderHasInternalSource(int channel, + unsigned int pl_type) const { + WEBRTC_ASSERT_CHANNEL(channel); + ASSERT(channels_.find(channel)->second->ext_encoders_.count(pl_type) != 0); + return channels_.find(channel) + ->second->ext_encoders_[pl_type] + .internal_source; }; int GetTotalNumExternalEncoderRegistered() const { std::map::const_iterator it; int total_num_registered = 0; for (it = channels_.begin(); it != channels_.end(); ++it) total_num_registered += - static_cast(it->second->ext_encoder_pl_types_.size()); + static_cast(it->second->ext_encoders_.size()); return total_num_registered; } void SetSendBitrates(int channel, unsigned int video_bitrate, @@ -1274,16 +1295,20 @@ class FakeWebRtcVideoEngine WEBRTC_VOID_STUB(DeRegisterPreRenderCallback, (int)); // webrtc::ViEExternalCodec WEBRTC_FUNC(RegisterExternalSendCodec, - (const int channel, const unsigned char pl_type, webrtc::VideoEncoder*, - bool)) { + (const int channel, + const unsigned char pl_type, + webrtc::VideoEncoder*, + bool internal_source)) { WEBRTC_CHECK_CHANNEL(channel); - channels_[channel]->ext_encoder_pl_types_.insert(pl_type); + ExternalEncoderInfo info; + info.internal_source = internal_source; + channels_[channel]->ext_encoders_[pl_type] = info; return 0; } WEBRTC_FUNC(DeRegisterExternalSendCodec, (const int channel, const unsigned char pl_type)) { WEBRTC_CHECK_CHANNEL(channel); - channels_[channel]->ext_encoder_pl_types_.erase(pl_type); + channels_[channel]->ext_encoders_.erase(pl_type); return 0; } WEBRTC_FUNC(RegisterExternalReceiveCodec, diff --git a/talk/media/webrtc/webrtcvideoencoderfactory.h b/talk/media/webrtc/webrtcvideoencoderfactory.h index b88c79a09..ad681fe66 100644 --- a/talk/media/webrtc/webrtcvideoencoderfactory.h +++ b/talk/media/webrtc/webrtcvideoencoderfactory.h @@ -63,6 +63,15 @@ class WebRtcVideoEncoderFactory { // Returns a list of supported codecs in order of preference. virtual const std::vector& codecs() const = 0; + // Returns true if encoders created by this factory of the given codec type + // will use internal camera sources, meaning that they don't require/expect + // frames to be delivered via webrtc::VideoEncoder::Encode. This flag is used + // as the internal_source parameter to + // webrtc::ViEExternalCodec::RegisterExternalSendCodec. + virtual bool EncoderTypeHasInternalSource(webrtc::VideoCodecType type) const { + return false; + } + virtual void DestroyVideoEncoder(webrtc::VideoEncoder* encoder) = 0; }; diff --git a/talk/media/webrtc/webrtcvideoengine.cc b/talk/media/webrtc/webrtcvideoengine.cc index 0e1ff527e..14264bb71 100644 --- a/talk/media/webrtc/webrtcvideoengine.cc +++ b/talk/media/webrtc/webrtcvideoengine.cc @@ -323,6 +323,11 @@ WebRtcSimulcastEncoderFactory::codecs() const { return factory_->codecs(); } +bool WebRtcSimulcastEncoderFactory::EncoderTypeHasInternalSource( + webrtc::VideoCodecType type) const { + return factory_->EncoderTypeHasInternalSource(type); +} + void WebRtcSimulcastEncoderFactory::DestroyVideoEncoder( webrtc::VideoEncoder* encoder) { // Check first to see if the encoder wasn't wrapped in a @@ -1623,10 +1628,13 @@ void WebRtcVideoEngine::DestroyExternalDecoder(webrtc::VideoDecoder* decoder) { } webrtc::VideoEncoder* WebRtcVideoEngine::CreateExternalEncoder( - webrtc::VideoCodecType type) { + webrtc::VideoCodecType type, + bool* internal_source) { + ASSERT(internal_source != NULL); if (!encoder_factory_) { return NULL; } + *internal_source = encoder_factory_->EncoderTypeHasInternalSource(type); return encoder_factory_->CreateVideoEncoder(type); } @@ -1912,8 +1920,9 @@ bool WebRtcVideoMediaChannel::MaybeRegisterExternalEncoder( return true; } + bool internal_source; webrtc::VideoEncoder* encoder = - engine()->CreateExternalEncoder(codec.codecType); + engine()->CreateExternalEncoder(codec.codecType, &internal_source); if (!encoder) { // No external encoder created, so nothing to do. return true; @@ -1921,7 +1930,7 @@ bool WebRtcVideoMediaChannel::MaybeRegisterExternalEncoder( const int channel_id = send_channel->channel_id(); if (engine()->vie()->ext_codec()->RegisterExternalSendCodec( - channel_id, codec.plType, encoder, false) != 0) { + channel_id, codec.plType, encoder, internal_source) != 0) { LOG_RTCERR2(RegisterExternalSendCodec, channel_id, codec.plName); engine()->DestroyExternalEncoder(encoder); return false; diff --git a/talk/media/webrtc/webrtcvideoengine.h b/talk/media/webrtc/webrtcvideoengine.h index 5004a382b..8fe6ada6f 100644 --- a/talk/media/webrtc/webrtcvideoengine.h +++ b/talk/media/webrtc/webrtcvideoengine.h @@ -152,7 +152,12 @@ class WebRtcVideoEngine : public sigslot::has_slots<> { // Returns an external encoder for the given codec type. The return value // can be NULL if encoder factory is not given or it does not support the // codec type. The caller takes the ownership of the returned object. - webrtc::VideoEncoder* CreateExternalEncoder(webrtc::VideoCodecType type); + // On success, |internal_source| is set to true if the encoder has an internal + // frame source, meaning that it doesn't expect/require frames through the + // normal camera pipeline. See ViEExternalCodec::RegisterExternalSendCodec for + // more information. + webrtc::VideoEncoder* CreateExternalEncoder(webrtc::VideoCodecType type, + bool* internal_source); // Releases the encoder instance created by CreateExternalEncoder(). void DestroyExternalEncoder(webrtc::VideoEncoder* encoder); @@ -534,6 +539,7 @@ class WebRtcSimulcastEncoderFactory webrtc::VideoEncoder* CreateVideoEncoder( webrtc::VideoCodecType type) override; const std::vector& codecs() const override; + bool EncoderTypeHasInternalSource(webrtc::VideoCodecType type) const override; void DestroyVideoEncoder(webrtc::VideoEncoder* encoder) override; private: diff --git a/talk/media/webrtc/webrtcvideoengine_unittest.cc b/talk/media/webrtc/webrtcvideoengine_unittest.cc index 18b38b639..a4501b321 100644 --- a/talk/media/webrtc/webrtcvideoengine_unittest.cc +++ b/talk/media/webrtc/webrtcvideoengine_unittest.cc @@ -1952,6 +1952,7 @@ TEST_F(WebRtcVideoEngineTestFake, DontRegisterEncoderIfFactoryIsNotGiven) { TEST_F(WebRtcVideoEngineTestFake, RegisterEncoderIfFactoryIsGiven) { encoder_factory_.AddSupportedVideoCodecType(webrtc::kVideoCodecVP8, "VP8"); + encoder_factory_.set_encoders_have_internal_sources(false); engine_.SetExternalEncoderFactory(&encoder_factory_); EXPECT_TRUE(SetupEngine()); int channel_num = vie_.GetLastChannel(); @@ -1964,6 +1965,29 @@ TEST_F(WebRtcVideoEngineTestFake, RegisterEncoderIfFactoryIsGiven) { cricket::StreamParams::CreateLegacy(kSsrc))); EXPECT_TRUE(vie_.ExternalEncoderRegistered(channel_num, 100)); + EXPECT_FALSE(vie_.ExternalEncoderHasInternalSource(channel_num, 100)); + EXPECT_EQ(1, vie_.GetNumExternalEncoderRegistered(channel_num)); + + // Remove stream previously added to free the external encoder instance. + EXPECT_TRUE(channel_->RemoveSendStream(kSsrc)); +} + +TEST_F(WebRtcVideoEngineTestFake, RegisterEncoderWithInternalSource) { + encoder_factory_.AddSupportedVideoCodecType(webrtc::kVideoCodecVP8, "VP8"); + encoder_factory_.set_encoders_have_internal_sources(true); + engine_.SetExternalEncoderFactory(&encoder_factory_); + EXPECT_TRUE(SetupEngine()); + int channel_num = vie_.GetLastChannel(); + + std::vector codecs; + codecs.push_back(kVP8Codec); + EXPECT_TRUE(channel_->SetSendCodecs(codecs)); + + EXPECT_TRUE( + channel_->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrc))); + + ASSERT_TRUE(vie_.ExternalEncoderRegistered(channel_num, 100)); + EXPECT_TRUE(vie_.ExternalEncoderHasInternalSource(channel_num, 100)); EXPECT_EQ(1, vie_.GetNumExternalEncoderRegistered(channel_num)); // Remove stream previously added to free the external encoder instance.