From c5300436841092d6d2b01eae1179a2c1eb373f1b Mon Sep 17 00:00:00 2001 From: "stefan@webrtc.org" Date: Mon, 8 Oct 2012 07:06:53 +0000 Subject: [PATCH] Add per stream intra requests. BUG= Review URL: https://webrtc-codereview.appspot.com/829006 git-svn-id: http://webrtc.googlecode.com/svn/trunk@2883 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../codecs/i420/main/interface/i420.h | 2 +- .../codecs/i420/main/source/i420.cc | 2 +- .../mock/mock_video_codec_interface.h | 2 +- .../codecs/interface/video_codec_interface.h | 14 ++- .../codecs/test/videoprocessor.cc | 6 +- .../test_framework/normal_async_test.cc | 9 +- .../codecs/test_framework/performance_test.cc | 6 +- .../codecs/test_framework/unit_test.cc | 38 +++--- .../video_coding/codecs/vp8/test/rps_test.cc | 3 +- src/modules/video_coding/codecs/vp8/vp8.gyp | 1 + .../video_coding/codecs/vp8/vp8_impl.cc | 8 +- .../video_coding/codecs/vp8/vp8_impl.h | 2 +- .../main/interface/video_coding.h | 2 +- .../video_coding/main/source/encoded_frame.cc | 39 +++--- .../video_coding/main/source/encoded_frame.h | 5 + .../main/source/generic_encoder.cc | 23 +++- .../main/source/generic_encoder.h | 4 +- .../main/source/video_coding_impl.cc | 25 ++-- .../main/source/video_coding_impl.h | 29 ++--- .../main/source/video_coding_impl_unittest.cc | 115 ++++++++++++++++++ .../main/source/video_coding_test.gypi | 1 + .../main/test/codec_database_test.cc | 6 +- .../test/libvietest/include/tb_I420_codec.h | 2 +- .../test/libvietest/testbed/tb_I420_codec.cc | 2 +- src/video_engine/vie_capturer.cc | 14 ++- src/video_engine/vie_capturer.h | 4 +- src/video_engine/vie_encoder.cc | 4 +- 27 files changed, 260 insertions(+), 108 deletions(-) create mode 100644 src/modules/video_coding/main/source/video_coding_impl_unittest.cc diff --git a/src/modules/video_coding/codecs/i420/main/interface/i420.h b/src/modules/video_coding/codecs/i420/main/interface/i420.h index ea740c55a..6699c2238 100644 --- a/src/modules/video_coding/codecs/i420/main/interface/i420.h +++ b/src/modules/video_coding/codecs/i420/main/interface/i420.h @@ -49,7 +49,7 @@ public: // <0 - Error virtual int Encode(const VideoFrame& inputImage, const CodecSpecificInfo* /*codecSpecificInfo*/, - const VideoFrameType /*frameTypes*/); + const std::vector* /*frame_types*/); // Register an encode complete callback object. // diff --git a/src/modules/video_coding/codecs/i420/main/source/i420.cc b/src/modules/video_coding/codecs/i420/main/source/i420.cc index 38cbbb878..75d85c141 100644 --- a/src/modules/video_coding/codecs/i420/main/source/i420.cc +++ b/src/modules/video_coding/codecs/i420/main/source/i420.cc @@ -78,7 +78,7 @@ int I420Encoder::InitEncode(const VideoCodec* codecSettings, int I420Encoder::Encode(const VideoFrame& inputImage, const CodecSpecificInfo* /*codecSpecificInfo*/, - const VideoFrameType /*frameType*/) { + const std::vector* /*frame_types*/) { if (!_inited) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } diff --git a/src/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h b/src/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h index 7b60cf5c2..f1123c922 100644 --- a/src/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h +++ b/src/modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h @@ -39,7 +39,7 @@ class MockVideoEncoder : public VideoEncoder { MOCK_METHOD3(Encode, WebRtc_Word32(const VideoFrame& inputImage, const CodecSpecificInfo* codecSpecificInfo, - const VideoFrameType frameType)); + const std::vector* frame_types)); MOCK_METHOD1(RegisterEncodeCompleteCallback, WebRtc_Word32(EncodedImageCallback* callback)); MOCK_METHOD0(Release, WebRtc_Word32()); diff --git a/src/modules/video_coding/codecs/interface/video_codec_interface.h b/src/modules/video_coding/codecs/interface/video_codec_interface.h index c107b18f5..7ae525336 100644 --- a/src/modules/video_coding/codecs/interface/video_codec_interface.h +++ b/src/modules/video_coding/codecs/interface/video_codec_interface.h @@ -11,6 +11,8 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_INTERFACE_VIDEO_CODEC_INTERFACE_H #define WEBRTC_MODULES_VIDEO_CODING_CODECS_INTERFACE_VIDEO_CODEC_INTERFACE_H +#include + #include "common_types.h" #include "modules/interface/module_common_types.h" #include "modules/video_coding/codecs/interface/video_error_codes.h" @@ -95,12 +97,14 @@ public: // Input: // - inputImage : Image to be encoded // - codecSpecificInfo : Pointer to codec specific data - // - frameType : The frame type to encode + // - frame_types : The frame type to encode // - // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. - virtual WebRtc_Word32 Encode(const VideoFrame& inputImage, - const CodecSpecificInfo* codecSpecificInfo, - const VideoFrameType frameType) = 0; + // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 + // otherwise. + virtual WebRtc_Word32 Encode( + const VideoFrame& inputImage, + const CodecSpecificInfo* codecSpecificInfo, + const std::vector* frame_types) = 0; // Register an encode complete callback object. // diff --git a/src/modules/video_coding/codecs/test/videoprocessor.cc b/src/modules/video_coding/codecs/test/videoprocessor.cc index fadc1e898..53bd11232 100644 --- a/src/modules/video_coding/codecs/test/videoprocessor.cc +++ b/src/modules/video_coding/codecs/test/videoprocessor.cc @@ -180,17 +180,17 @@ bool VideoProcessorImpl::ProcessFrame(int frame_number) { source_frame_.SetTimeStamp(frame_number); // Decide if we're going to force a keyframe: - VideoFrameType frame_type = kDeltaFrame; + std::vector frame_types(1, kDeltaFrame); if (config_.keyframe_interval > 0 && frame_number % config_.keyframe_interval == 0) { - frame_type = kKeyFrame; + frame_types[0] = kKeyFrame; } // For dropped frames, we regard them as zero size encoded frames. encoded_frame_size_ = 0; WebRtc_Word32 encode_result = encoder_->Encode(source_frame_, NULL, - frame_type); + &frame_types); if (encode_result != WEBRTC_VIDEO_CODEC_OK) { fprintf(stderr, "Failed to encode frame %d, return code: %d\n", diff --git a/src/modules/video_coding/codecs/test_framework/normal_async_test.cc b/src/modules/video_coding/codecs/test_framework/normal_async_test.cc index c9081d9f5..4a35d1a5f 100644 --- a/src/modules/video_coding/codecs/test_framework/normal_async_test.cc +++ b/src/modules/video_coding/codecs/test_framework/normal_async_test.cc @@ -12,8 +12,9 @@ #include #include -#include #include +#include +#include #include "gtest/gtest.h" #include "tick_util.h" @@ -422,7 +423,7 @@ NormalAsyncTest::Encode() } _encodeCompleteTime = 0; _encodeTimes[rawImage.TimeStamp()] = tGetTime(); - VideoFrameType frameType = kDeltaFrame; + std::vector frame_types(1, kDeltaFrame); // check SLI queue _hasReceivedSLI = false; @@ -458,13 +459,13 @@ NormalAsyncTest::Encode() if (_hasReceivedPLI) { // respond to PLI by encoding a key frame - frameType = kKeyFrame; + frame_types[0] = kKeyFrame; _hasReceivedPLI = false; _hasReceivedSLI = false; // don't trigger both at once } webrtc::CodecSpecificInfo* codecSpecificInfo = CreateEncoderSpecificInfo(); - int ret = _encoder->Encode(rawImage, codecSpecificInfo, frameType); + int ret = _encoder->Encode(rawImage, codecSpecificInfo, &frame_types); EXPECT_EQ(ret, WEBRTC_VIDEO_CODEC_OK); if (codecSpecificInfo != NULL) { diff --git a/src/modules/video_coding/codecs/test_framework/performance_test.cc b/src/modules/video_coding/codecs/test_framework/performance_test.cc index 18c6ad9b8..d23592899 100644 --- a/src/modules/video_coding/codecs/test_framework/performance_test.cc +++ b/src/modules/video_coding/codecs/test_framework/performance_test.cc @@ -267,13 +267,13 @@ bool PerformanceTest::Encode() { VideoFrame rawImage; VideoBufferToRawImage(_inputVideoBuffer, rawImage); - VideoFrameType frameType = kDeltaFrame; + std::vector frameTypes(1, kDeltaFrame); if (_requestKeyFrame && !(_encFrameCnt%50)) { - frameType = kKeyFrame; + frameTypes[0] = kKeyFrame; } webrtc::CodecSpecificInfo* codecSpecificInfo = CreateEncoderSpecificInfo(); - int ret = _encoder->Encode(rawImage, codecSpecificInfo, frameType); + int ret = _encoder->Encode(rawImage, codecSpecificInfo, &frameTypes); EXPECT_EQ(ret, WEBRTC_VIDEO_CODEC_OK); if (codecSpecificInfo != NULL) { diff --git a/src/modules/video_coding/codecs/test_framework/unit_test.cc b/src/modules/video_coding/codecs/test_framework/unit_test.cc index dbbbcf0ae..4863f0e8f 100644 --- a/src/modules/video_coding/codecs/test_framework/unit_test.cc +++ b/src/modules/video_coding/codecs/test_framework/unit_test.cc @@ -240,8 +240,7 @@ UnitTest::Setup() // Ensures our initial parameters are valid. EXPECT_TRUE(_encoder->InitEncode(&_inst, 1, 1440) == WEBRTC_VIDEO_CODEC_OK); - VideoFrameType videoFrameType = kDeltaFrame; - _encoder->Encode(image, NULL, videoFrameType); + _encoder->Encode(image, NULL, NULL); _refEncFrameLength = WaitForEncodedFrame(); ASSERT_TRUE(_refEncFrameLength > 0); _refEncFrame = new unsigned char[_refEncFrameLength]; @@ -266,7 +265,7 @@ UnitTest::Setup() _inputVideoBuffer.SetWidth(_source->GetWidth()); _inputVideoBuffer.SetHeight(_source->GetHeight()); VideoBufferToRawImage(_inputVideoBuffer, image); - _encoder->Encode(image, NULL, videoFrameType); + _encoder->Encode(image, NULL, NULL); ASSERT_TRUE(WaitForEncodedFrame() > 0); } EncodedImage encodedImage; @@ -352,7 +351,6 @@ UnitTest::Perform() int frameLength; VideoFrame inputImage; EncodedImage encodedImage; - VideoFrameType videoFrameType = kDeltaFrame; //----- Encoder parameter tests ----- @@ -360,7 +358,7 @@ UnitTest::Perform() // We want to revert the initialization done in Setup(). EXPECT_TRUE(_encoder->Release() == WEBRTC_VIDEO_CODEC_OK); VideoBufferToRawImage(_inputVideoBuffer, inputImage); - EXPECT_TRUE(_encoder->Encode(inputImage, NULL, videoFrameType) + EXPECT_TRUE(_encoder->Encode(inputImage, NULL, NULL) == WEBRTC_VIDEO_CODEC_UNINITIALIZED); //-- InitEncode() errors -- @@ -423,7 +421,7 @@ UnitTest::Perform() // inputVideoBuffer unallocated. _inputVideoBuffer.Free(); inputImage.Free(); - EXPECT_TRUE(_encoder->Encode(inputImage, NULL, videoFrameType) == + EXPECT_TRUE(_encoder->Encode(inputImage, NULL, NULL) == WEBRTC_VIDEO_CODEC_ERR_PARAMETER); _inputVideoBuffer.VerifyAndAllocate(_lengthSourceFrame); _inputVideoBuffer.CopyBuffer(_lengthSourceFrame, _refFrame); @@ -436,8 +434,9 @@ UnitTest::Perform() VideoBufferToRawImage(_inputVideoBuffer, inputImage); for (int i = 1; i <= 60; i++) { - VideoFrameType frameType = !(i % 2) ? kKeyFrame : kDeltaFrame; - EXPECT_TRUE(_encoder->Encode(inputImage, NULL, frameType) == + VideoFrameType frame_type = !(i % 2) ? kKeyFrame : kDeltaFrame; + std::vector frame_types(1, frame_type); + EXPECT_TRUE(_encoder->Encode(inputImage, NULL, &frame_types) == WEBRTC_VIDEO_CODEC_OK); EXPECT_TRUE(WaitForEncodedFrame() > 0); } @@ -445,12 +444,12 @@ UnitTest::Perform() // Init then encode. _encodedVideoBuffer.UpdateLength(0); _encodedVideoBuffer.Reset(); - EXPECT_TRUE(_encoder->Encode(inputImage, NULL, videoFrameType) == + EXPECT_TRUE(_encoder->Encode(inputImage, NULL, NULL) == WEBRTC_VIDEO_CODEC_OK); EXPECT_TRUE(WaitForEncodedFrame() > 0); EXPECT_TRUE(_encoder->InitEncode(&_inst, 1, 1440) == WEBRTC_VIDEO_CODEC_OK); - _encoder->Encode(inputImage, NULL, videoFrameType); + _encoder->Encode(inputImage, NULL, NULL); frameLength = WaitForEncodedFrame(); EXPECT_TRUE(frameLength > 0); EXPECT_TRUE(CheckIfBitExact(_refEncFrame, _refEncFrameLength, @@ -459,11 +458,11 @@ UnitTest::Perform() // Reset then encode. _encodedVideoBuffer.UpdateLength(0); _encodedVideoBuffer.Reset(); - EXPECT_TRUE(_encoder->Encode(inputImage, NULL, videoFrameType) == + EXPECT_TRUE(_encoder->Encode(inputImage, NULL, NULL) == WEBRTC_VIDEO_CODEC_OK); WaitForEncodedFrame(); EXPECT_TRUE(_encoder->InitEncode(&_inst, 1, 1440) == WEBRTC_VIDEO_CODEC_OK); - _encoder->Encode(inputImage, NULL, videoFrameType); + _encoder->Encode(inputImage, NULL, NULL); frameLength = WaitForEncodedFrame(); EXPECT_TRUE(frameLength > 0); EXPECT_TRUE(CheckIfBitExact(_refEncFrame, _refEncFrameLength, @@ -472,12 +471,12 @@ UnitTest::Perform() // Release then encode. _encodedVideoBuffer.UpdateLength(0); _encodedVideoBuffer.Reset(); - EXPECT_TRUE(_encoder->Encode(inputImage, NULL, videoFrameType) == + EXPECT_TRUE(_encoder->Encode(inputImage, NULL, NULL) == WEBRTC_VIDEO_CODEC_OK); WaitForEncodedFrame(); EXPECT_TRUE(_encoder->Release() == WEBRTC_VIDEO_CODEC_OK); EXPECT_TRUE(_encoder->InitEncode(&_inst, 1, 1440) == WEBRTC_VIDEO_CODEC_OK); - _encoder->Encode(inputImage, NULL, videoFrameType); + _encoder->Encode(inputImage, NULL, NULL); frameLength = WaitForEncodedFrame(); EXPECT_TRUE(frameLength > 0); EXPECT_TRUE(CheckIfBitExact(_refEncFrame, _refEncFrameLength, @@ -588,8 +587,7 @@ UnitTest::Perform() tempInput.CopyFrame(tmpLength, inputImage.Buffer()); tempInput.SetWidth(tempInst.width); tempInput.SetHeight(tempInst.height); - VideoFrameType videoFrameType = kDeltaFrame; - _encoder->Encode(tempInput, NULL, videoFrameType); + _encoder->Encode(tempInput, NULL, NULL); frameLength = WaitForEncodedFrame(); EXPECT_TRUE(frameLength > 0); tempInput.Free(); @@ -607,7 +605,7 @@ UnitTest::Perform() EXPECT_TRUE(_encoder->Release() == WEBRTC_VIDEO_CODEC_OK); EXPECT_TRUE(_encoder->InitEncode(&_inst, 1, 1440) == WEBRTC_VIDEO_CODEC_OK); - _encoder->Encode(inputImage, NULL, videoFrameType); + _encoder->Encode(inputImage, NULL, NULL); frameLength = WaitForEncodedFrame(); EXPECT_TRUE(frameLength > 0); @@ -666,8 +664,7 @@ UnitTest::Perform() _inputVideoBuffer.CopyBuffer(_lengthSourceFrame, _sourceBuffer); _inputVideoBuffer.SetTimeStamp(frames); VideoBufferToRawImage(_inputVideoBuffer, inputImage); - VideoFrameType videoFrameType = kDeltaFrame; - ASSERT_TRUE(_encoder->Encode(inputImage, NULL, videoFrameType) == + ASSERT_TRUE(_encoder->Encode(inputImage, NULL, NULL) == WEBRTC_VIDEO_CODEC_OK); frameLength = WaitForEncodedFrame(); //ASSERT_TRUE(frameLength); @@ -745,8 +742,7 @@ UnitTest::RateControlTests() static_cast(9e4 / static_cast(_inst.maxFramerate))); VideoBufferToRawImage(_inputVideoBuffer, inputImage); - VideoFrameType videoFrameType = kDeltaFrame; - ASSERT_EQ(_encoder->Encode(inputImage, NULL, videoFrameType), + ASSERT_EQ(_encoder->Encode(inputImage, NULL, NULL), WEBRTC_VIDEO_CODEC_OK); frameLength = WaitForEncodedFrame(); ASSERT_GE(frameLength, 0u); diff --git a/src/modules/video_coding/codecs/vp8/test/rps_test.cc b/src/modules/video_coding/codecs/vp8/test/rps_test.cc index 82b63db50..dd27f4c3b 100644 --- a/src/modules/video_coding/codecs/vp8/test/rps_test.cc +++ b/src/modules/video_coding/codecs/vp8/test/rps_test.cc @@ -149,7 +149,6 @@ bool VP8RpsTest::EncodeRps(RpsDecodeCompleteCallback* decodeCallback) { } _encodeCompleteTime = 0; _encodeTimes[rawImage.TimeStamp()] = tGetTime(); - webrtc::VideoFrameType frameType = webrtc::kDeltaFrame; webrtc::CodecSpecificInfo* codecSpecificInfo = CreateEncoderSpecificInfo(); codecSpecificInfo->codecSpecific.VP8.pictureIdRPSI = @@ -162,7 +161,7 @@ bool VP8RpsTest::EncodeRps(RpsDecodeCompleteCallback* decodeCallback) { sli_ = false; } printf("Encoding: %u\n", _framecnt); - int ret = _encoder->Encode(rawImage, codecSpecificInfo, frameType); + int ret = _encoder->Encode(rawImage, codecSpecificInfo, NULL); if (ret < 0) printf("Failed to encode: %u\n", _framecnt); diff --git a/src/modules/video_coding/codecs/vp8/vp8.gyp b/src/modules/video_coding/codecs/vp8/vp8.gyp index c828d6ad1..373a8640a 100644 --- a/src/modules/video_coding/codecs/vp8/vp8.gyp +++ b/src/modules/video_coding/codecs/vp8/vp8.gyp @@ -103,6 +103,7 @@ 'type': 'executable', 'dependencies': [ 'webrtc_vp8', + '<(DEPTH)/testing/gmock.gyp:gmock', '<(DEPTH)/testing/gtest.gyp:gtest', '<(webrtc_root)/test/test.gyp:test_support_main', ], diff --git a/src/modules/video_coding/codecs/vp8/vp8_impl.cc b/src/modules/video_coding/codecs/vp8/vp8_impl.cc index 4f08f4658..5f5a6bc18 100644 --- a/src/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/src/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -326,7 +326,7 @@ uint32_t VP8EncoderImpl::MaxIntraTarget(uint32_t optimalBuffersize) { int VP8EncoderImpl::Encode(const VideoFrame& input_image, const CodecSpecificInfo* codec_specific_info, - const VideoFrameType frame_type) { + const std::vector* frame_types) { if (!inited_) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } @@ -337,6 +337,12 @@ int VP8EncoderImpl::Encode(const VideoFrame& input_image, return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } + VideoFrameType frame_type = kDeltaFrame; + // We only support one stream at the moment. + if (frame_types && frame_types->size() > 0) { + frame_type = (*frame_types)[0]; + } + // Check for change in frame size. if (input_image.Width() != codec_.width || input_image.Height() != codec_.height) { diff --git a/src/modules/video_coding/codecs/vp8/vp8_impl.h b/src/modules/video_coding/codecs/vp8/vp8_impl.h index 82c85eac8..e1843a665 100644 --- a/src/modules/video_coding/codecs/vp8/vp8_impl.h +++ b/src/modules/video_coding/codecs/vp8/vp8_impl.h @@ -74,7 +74,7 @@ class VP8EncoderImpl : public VP8Encoder { virtual int Encode(const VideoFrame& input_image, const CodecSpecificInfo* codec_specific_info, - const VideoFrameType frame_type); + const std::vector* frame_types); // Register an encode complete callback object. // diff --git a/src/modules/video_coding/main/interface/video_coding.h b/src/modules/video_coding/main/interface/video_coding.h index af054912f..bea0107d4 100644 --- a/src/modules/video_coding/main/interface/video_coding.h +++ b/src/modules/video_coding/main/interface/video_coding.h @@ -260,7 +260,7 @@ public: // // Return value : VCM_OK, on success. // < 0, on error. - virtual WebRtc_Word32 IntraFrameRequest() = 0; + virtual WebRtc_Word32 IntraFrameRequest(int stream_index) = 0; // Frame Dropper enable. Can be used to disable the frame dropping when the encoder // over-uses its bit rate. This API is designed to be used when the encoded frames diff --git a/src/modules/video_coding/main/source/encoded_frame.cc b/src/modules/video_coding/main/source/encoded_frame.cc index dff9df3ee..a7e9bb70e 100644 --- a/src/modules/video_coding/main/source/encoded_frame.cc +++ b/src/modules/video_coding/main/source/encoded_frame.cc @@ -227,31 +227,30 @@ webrtc::FrameType VCMEncodedFrame::ConvertFrameType(VideoFrameType frameType) } } -VideoFrameType VCMEncodedFrame::ConvertFrameType(webrtc::FrameType frameType) -{ - switch (frameType) - { +VideoFrameType VCMEncodedFrame::ConvertFrameType(webrtc::FrameType frame_type) { + switch (frame_type) { case kVideoFrameKey: - { - return kKeyFrame; - } + return kKeyFrame; case kVideoFrameDelta: - { - return kDeltaFrame; - } + return kDeltaFrame; case kVideoFrameGolden: - { - return kGoldenFrame; - } + return kGoldenFrame; case kVideoFrameAltRef: - { - return kAltRefFrame; - } + return kAltRefFrame; default: - { - return kDeltaFrame; - } - } + assert(false); + return kDeltaFrame; + } +} + +void VCMEncodedFrame::ConvertFrameTypes( + const std::vector& frame_types, + std::vector* video_frame_types) { + assert(video_frame_types); + video_frame_types->reserve(frame_types.size()); + for (size_t i = 0; i < frame_types.size(); ++i) { + (*video_frame_types)[i] = ConvertFrameType(frame_types[i]); + } } } diff --git a/src/modules/video_coding/main/source/encoded_frame.h b/src/modules/video_coding/main/source/encoded_frame.h index 6289e9ecc..932e98bb5 100644 --- a/src/modules/video_coding/main/source/encoded_frame.h +++ b/src/modules/video_coding/main/source/encoded_frame.h @@ -11,6 +11,8 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_ENCODED_FRAME_H_ #define WEBRTC_MODULES_VIDEO_CODING_ENCODED_FRAME_H_ +#include + #include "common_types.h" #include "common_video/interface/video_image.h" #include "modules/interface/module_common_types.h" @@ -93,6 +95,9 @@ public: static webrtc::FrameType ConvertFrameType(VideoFrameType frameType); static VideoFrameType ConvertFrameType(webrtc::FrameType frameType); + static void ConvertFrameTypes( + const std::vector& frame_types, + std::vector* video_frame_types); protected: /** diff --git a/src/modules/video_coding/main/source/generic_encoder.cc b/src/modules/video_coding/main/source/generic_encoder.cc index 8ead0e528..1bf8fbee8 100644 --- a/src/modules/video_coding/main/source/generic_encoder.cc +++ b/src/modules/video_coding/main/source/generic_encoder.cc @@ -59,9 +59,13 @@ VCMGenericEncoder::InitEncode(const VideoCodec* settings, WebRtc_Word32 VCMGenericEncoder::Encode(const VideoFrame& inputFrame, const CodecSpecificInfo* codecSpecificInfo, - const FrameType frameType) { - VideoFrameType videoFrameType = VCMEncodedFrame::ConvertFrameType(frameType); - return _encoder.Encode(inputFrame, codecSpecificInfo, videoFrameType); + const std::vector* frameTypes) { + std::vector video_frame_types(frameTypes->size(), + kDeltaFrame); + if (frameTypes) { + VCMEncodedFrame::ConvertFrameTypes(*frameTypes, &video_frame_types); + } + return _encoder.Encode(inputFrame, codecSpecificInfo, &video_frame_types); } WebRtc_Word32 @@ -110,10 +114,17 @@ VCMGenericEncoder::SetPeriodicKeyFrames(bool enable) return _encoder.SetPeriodicKeyFrames(enable); } -WebRtc_Word32 VCMGenericEncoder::RequestFrame(const FrameType frameType) { +WebRtc_Word32 VCMGenericEncoder::RequestFrame( + const std::vector* frame_types) { + if (!frame_types) { + return 0; + } VideoFrame image; - VideoFrameType videoFrameType = VCMEncodedFrame::ConvertFrameType(frameType); - return _encoder.Encode(image, NULL, videoFrameType); + std::vector video_frame_types(kVideoFrameDelta); + if (frame_types) { + VCMEncodedFrame::ConvertFrameTypes(*frame_types, &video_frame_types); + } + return _encoder.Encode(image, NULL, &video_frame_types); } WebRtc_Word32 diff --git a/src/modules/video_coding/main/source/generic_encoder.h b/src/modules/video_coding/main/source/generic_encoder.h index c75339b7c..9e8ae1694 100644 --- a/src/modules/video_coding/main/source/generic_encoder.h +++ b/src/modules/video_coding/main/source/generic_encoder.h @@ -101,7 +101,7 @@ public: */ WebRtc_Word32 Encode(const VideoFrame& inputFrame, const CodecSpecificInfo* codecSpecificInfo, - const FrameType frameType); + const std::vector* frameTypes); /** * Set new target bit rate and frame rate * Return Value: new bit rate if OK, otherwise <0s @@ -127,7 +127,7 @@ public: WebRtc_Word32 SetPeriodicKeyFrames(bool enable); - WebRtc_Word32 RequestFrame(const FrameType frameType); + WebRtc_Word32 RequestFrame(const std::vector* frame_types); bool InternalSource() const; diff --git a/src/modules/video_coding/main/source/video_coding_impl.cc b/src/modules/video_coding/main/source/video_coding_impl.cc index a0c853f75..ded4fb7d2 100644 --- a/src/modules/video_coding/main/source/video_coding_impl.cc +++ b/src/modules/video_coding/main/source/video_coding_impl.cc @@ -73,7 +73,7 @@ _scheduleKeyRequest(false), _sendCritSect(CriticalSectionWrapper::CreateCriticalSection()), _encoder(), _encodedFrameCallback(), -_nextFrameType(kVideoFrameDelta), +_nextFrameTypes(1, kVideoFrameDelta), _mediaOpt(id, clock_), _sendCodecType(kVideoCodecUnknown), _sendStatsCallback(NULL), @@ -334,6 +334,9 @@ VideoCodingModuleImpl::RegisterSendCodec(const VideoCodec* sendCodec, _sendCodecType = sendCodec->codecType; int numLayers = (_sendCodecType != kVideoCodecVP8) ? 1 : sendCodec->codecSpecific.VP8.numberOfTemporalLayers; + _nextFrameTypes.clear(); + _nextFrameTypes.resize(VCM_MAX(sendCodec->numberOfSimulcastStreams, 1), + kVideoFrameDelta); _mediaOpt.SetEncodingData(_sendCodecType, sendCodec->maxBitrate, @@ -661,7 +664,9 @@ VideoCodingModuleImpl::AddVideoFrame(const VideoFrame& videoFrame, { return VCM_UNINITIALIZED; } - if (_nextFrameType == kFrameEmpty) + // TODO(holmer): Add support for dropping frames per stream. Currently we + // only have one frame dropper for all streams. + if (_nextFrameTypes[0] == kFrameEmpty) { return VCM_OK; } @@ -679,7 +684,7 @@ VideoCodingModuleImpl::AddVideoFrame(const VideoFrame& videoFrame, _mediaOpt.updateContentData(contentMetrics); WebRtc_Word32 ret = _encoder->Encode(videoFrame, codecSpecificInfo, - _nextFrameType); + &_nextFrameTypes); if (_encoderInputFile != NULL) { if (fwrite(videoFrame.Buffer(), 1, videoFrame.Length(), @@ -695,19 +700,23 @@ VideoCodingModuleImpl::AddVideoFrame(const VideoFrame& videoFrame, "Encode error: %d", ret); return ret; } - _nextFrameType = kVideoFrameDelta; // default frame type + for (size_t i = 0; i < _nextFrameTypes.size(); ++i) { + _nextFrameTypes[i] = kVideoFrameDelta; // Default frame type. + } } return VCM_OK; } -WebRtc_Word32 VideoCodingModuleImpl::IntraFrameRequest() { +WebRtc_Word32 VideoCodingModuleImpl::IntraFrameRequest(int stream_index) { + assert(stream_index >= 0); CriticalSectionScoped cs(_sendCritSect); - _nextFrameType = kVideoFrameKey; + _nextFrameTypes[stream_index] = kVideoFrameKey; if (_encoder != NULL && _encoder->InternalSource()) { // Try to request the frame if we have an external encoder with // internal source since AddVideoFrame never will be called. - if (_encoder->RequestFrame(_nextFrameType) == WEBRTC_VIDEO_CODEC_OK) { - _nextFrameType = kVideoFrameDelta; + if (_encoder->RequestFrame(&_nextFrameTypes) == + WEBRTC_VIDEO_CODEC_OK) { + _nextFrameTypes[stream_index] = kVideoFrameDelta; } } return VCM_OK; diff --git a/src/modules/video_coding/main/source/video_coding_impl.h b/src/modules/video_coding/main/source/video_coding_impl.h index 1e39cbbec..ed42ced5b 100644 --- a/src/modules/video_coding/main/source/video_coding_impl.h +++ b/src/modules/video_coding/main/source/video_coding_impl.h @@ -11,19 +11,20 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_ #define WEBRTC_MODULES_VIDEO_CODING_VIDEO_CODING_IMPL_H_ -#include "video_coding.h" -#include "critical_section_wrapper.h" -#include "frame_buffer.h" -#include "receiver.h" -#include "timing.h" -#include "jitter_buffer.h" -#include "codec_database.h" -#include "generic_decoder.h" -#include "generic_encoder.h" -#include "media_optimization.h" -#include "modules/video_coding/main/source/tick_time_base.h" +#include "modules/video_coding/main/interface/video_coding.h" -#include +#include + +#include "modules/video_coding/main/source/codec_database.h" +#include "modules/video_coding/main/source/frame_buffer.h" +#include "modules/video_coding/main/source/generic_decoder.h" +#include "modules/video_coding/main/source/generic_encoder.h" +#include "modules/video_coding/main/source/jitter_buffer.h" +#include "modules/video_coding/main/source/media_optimization.h" +#include "modules/video_coding/main/source/receiver.h" +#include "modules/video_coding/main/source/tick_time_base.h" +#include "modules/video_coding/main/source/timing.h" +#include "system_wrappers/interface/critical_section_wrapper.h" namespace webrtc { @@ -147,7 +148,7 @@ public: const VideoContentMetrics* _contentMetrics = NULL, const CodecSpecificInfo* codecSpecificInfo = NULL); - virtual WebRtc_Word32 IntraFrameRequest(); + virtual WebRtc_Word32 IntraFrameRequest(int stream_index); //Enable frame dropper virtual WebRtc_Word32 EnableFrameDropper(bool enable); @@ -300,7 +301,7 @@ private: CriticalSectionWrapper* _sendCritSect; // Critical section for send side VCMGenericEncoder* _encoder; VCMEncodedFrameCallback _encodedFrameCallback; - FrameType _nextFrameType; + std::vector _nextFrameTypes; VCMMediaOptimization _mediaOpt; VideoCodecType _sendCodecType; VCMSendStatisticsCallback* _sendStatsCallback; diff --git a/src/modules/video_coding/main/source/video_coding_impl_unittest.cc b/src/modules/video_coding/main/source/video_coding_impl_unittest.cc new file mode 100644 index 000000000..bfb8227e4 --- /dev/null +++ b/src/modules/video_coding/main/source/video_coding_impl_unittest.cc @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "modules/video_coding/codecs/interface/mock/mock_video_codec_interface.h" +#include "modules/video_coding/main/interface/video_coding.h" +#include "system_wrappers/interface/scoped_ptr.h" + +#include "gtest/gtest.h" + +using ::testing::_; +using ::testing::AllOf; +using ::testing::ElementsAre; +using ::testing::ElementsAreArray; +using ::testing::Field; +using ::testing::NiceMock; +using ::testing::Pointee; +using ::testing::Return; + +namespace webrtc { + +class TestVideoCodingModule : public ::testing::Test { + protected: + static const int kDefaultWidth = 1280; + static const int kDefaultHeight = 720; + static const int kNumberOfStreams = 3; + static const int kNumberOfLayers = 3; + static const int kUnusedPayloadType = 10; + + virtual void SetUp() { + vcm_ = VideoCodingModule::Create(0); + EXPECT_EQ(0, vcm_->RegisterExternalEncoder(&encoder_, kUnusedPayloadType, + false)); + memset(&settings_, 0, sizeof(settings_)); + EXPECT_EQ(0, vcm_->Codec(kVideoCodecVP8, &settings_)); + settings_.numberOfSimulcastStreams = kNumberOfStreams; + ConfigureStream(kDefaultWidth / 4, kDefaultHeight / 4, 100, + &settings_.simulcastStream[0]); + ConfigureStream(kDefaultWidth / 2, kDefaultHeight / 2, 500, + &settings_.simulcastStream[1]); + ConfigureStream(kDefaultWidth, kDefaultHeight, 1200, + &settings_.simulcastStream[2]); + settings_.plType = kUnusedPayloadType; // Use the mocked encoder. + EXPECT_EQ(0, vcm_->RegisterSendCodec(&settings_, 1, 1200)); + } + + virtual void TearDown() { + VideoCodingModule::Destroy(vcm_); + input_frame_.Free(); + } + + void ExpectIntraRequest(int stream) { + if (stream == -1) { + // No intra request expected. + EXPECT_CALL(encoder_, Encode( + _, _, Pointee(ElementsAre(kDeltaFrame, kDeltaFrame, kDeltaFrame)))) + .Times(1) + .WillRepeatedly(Return(0)); + return; + } + assert(stream >= 0); + assert(stream < kNumberOfStreams); + std::vector frame_types(kNumberOfStreams, kDeltaFrame); + frame_types[stream] = kKeyFrame; + EXPECT_CALL(encoder_, Encode( + _, _, Pointee(ElementsAreArray(&frame_types[0], frame_types.size())))) + .Times(1) + .WillRepeatedly(Return(0)); + } + + static void ConfigureStream(int width, int height, int max_bitrate, + SimulcastStream* stream) { + assert(stream); + stream->width = width; + stream->height = height; + stream->maxBitrate = max_bitrate; + stream->numberOfTemporalLayers = kNumberOfLayers; + stream->qpMax = 45; + } + + VideoCodingModule* vcm_; + NiceMock encoder_; + VideoFrame input_frame_; + VideoCodec settings_; +}; + +TEST_F(TestVideoCodingModule, TestIntraRequests) { + EXPECT_EQ(0, vcm_->IntraFrameRequest(0)); + ExpectIntraRequest(0); + EXPECT_EQ(0, vcm_->AddVideoFrame(input_frame_, NULL, NULL)); + ExpectIntraRequest(-1); + EXPECT_EQ(0, vcm_->AddVideoFrame(input_frame_, NULL, NULL)); + + EXPECT_EQ(0, vcm_->IntraFrameRequest(1)); + ExpectIntraRequest(1); + EXPECT_EQ(0, vcm_->AddVideoFrame(input_frame_, NULL, NULL)); + ExpectIntraRequest(-1); + EXPECT_EQ(0, vcm_->AddVideoFrame(input_frame_, NULL, NULL)); + + EXPECT_EQ(0, vcm_->IntraFrameRequest(2)); + ExpectIntraRequest(2); + EXPECT_EQ(0, vcm_->AddVideoFrame(input_frame_, NULL, NULL)); + ExpectIntraRequest(-1); + EXPECT_EQ(0, vcm_->AddVideoFrame(input_frame_, NULL, NULL)); +} + +} // namespace webrtc diff --git a/src/modules/video_coding/main/source/video_coding_test.gypi b/src/modules/video_coding/main/source/video_coding_test.gypi index 72e9ad370..249f7a4f4 100644 --- a/src/modules/video_coding/main/source/video_coding_test.gypi +++ b/src/modules/video_coding/main/source/video_coding_test.gypi @@ -83,6 +83,7 @@ 'jitter_buffer_unittest.cc', 'session_info_unittest.cc', 'video_coding_robustness_unittest.cc', + 'video_coding_impl_unittest.cc', 'qm_select_unittest.cc', ], }, diff --git a/src/modules/video_coding/main/test/codec_database_test.cc b/src/modules/video_coding/main/test/codec_database_test.cc index 10487bd01..7696300f0 100644 --- a/src/modules/video_coding/main/test/codec_database_test.cc +++ b/src/modules/video_coding/main/test/codec_database_test.cc @@ -238,7 +238,7 @@ CodecDataBaseTest::Perform(CmdArgs& args) // Try to decode a delta frame. Should get a warning since we have enabled the "require key frame" setting // and because no frame type request callback has been registered. TEST(_vcm->Decode() == VCM_MISSING_CALLBACK); - TEST(_vcm->IntraFrameRequest() == VCM_OK); + TEST(_vcm->IntraFrameRequest(0) == VCM_OK); _timeStamp += (WebRtc_UWord32)(9e4 / _frameRate); sourceFrame.SetTimeStamp(_timeStamp); TEST(_vcm->AddVideoFrame(sourceFrame) == VCM_OK); @@ -250,7 +250,7 @@ CodecDataBaseTest::Perform(CmdArgs& args) sendCodec.width = _width; sendCodec.height = _height; TEST(_vcm->RegisterReceiveCodec(&sendCodec, 1) == VCM_OK); - TEST(_vcm->IntraFrameRequest() == VCM_OK); + TEST(_vcm->IntraFrameRequest(0) == VCM_OK); waitEvent->Wait(33); _timeStamp += (WebRtc_UWord32)(9e4 / _frameRate); sourceFrame.SetTimeStamp(_timeStamp); @@ -260,7 +260,7 @@ CodecDataBaseTest::Perform(CmdArgs& args) waitEvent->Wait(33); _timeStamp += (WebRtc_UWord32)(9e4 / _frameRate); sourceFrame.SetTimeStamp(_timeStamp); - TEST(_vcm->IntraFrameRequest() == VCM_OK); + TEST(_vcm->IntraFrameRequest(0) == VCM_OK); TEST(_vcm->AddVideoFrame(sourceFrame) == VCM_OK); TEST(_vcm->Decode() == VCM_OK); TEST(_vcm->ResetDecoder() == VCM_OK); diff --git a/src/video_engine/test/libvietest/include/tb_I420_codec.h b/src/video_engine/test/libvietest/include/tb_I420_codec.h index e1c9b7980..0d1521238 100644 --- a/src/video_engine/test/libvietest/include/tb_I420_codec.h +++ b/src/video_engine/test/libvietest/include/tb_I420_codec.h @@ -36,7 +36,7 @@ public: virtual WebRtc_Word32 Encode( const webrtc::VideoFrame& inputImage, const webrtc::CodecSpecificInfo* codecSpecificInfo, - const webrtc::VideoFrameType frameType); + const std::vector* frameTypes); virtual WebRtc_Word32 RegisterEncodeCompleteCallback( webrtc::EncodedImageCallback* callback); diff --git a/src/video_engine/test/libvietest/testbed/tb_I420_codec.cc b/src/video_engine/test/libvietest/testbed/tb_I420_codec.cc index af30307eb..2782f93b9 100644 --- a/src/video_engine/test/libvietest/testbed/tb_I420_codec.cc +++ b/src/video_engine/test/libvietest/testbed/tb_I420_codec.cc @@ -120,7 +120,7 @@ WebRtc_Word32 TbI420Encoder::InitEncode(const webrtc::VideoCodec* inst, WebRtc_Word32 TbI420Encoder::Encode( const webrtc::VideoFrame& inputImage, const webrtc::CodecSpecificInfo* /*codecSpecificInfo*/, - const webrtc::VideoFrameType /*frameType*/) + const std::vector* /*frameTypes*/) { _functionCalls.Encode++; if (!_inited) diff --git a/src/video_engine/vie_capturer.cc b/src/video_engine/vie_capturer.cc index 58044754a..09c09db94 100644 --- a/src/video_engine/vie_capturer.cc +++ b/src/video_engine/vie_capturer.cc @@ -749,17 +749,19 @@ WebRtc_Word32 ViECapturer::InitEncode(const VideoCodec* codec_settings, return capture_encoder_->ConfigureEncoder(*codec_settings, max_payload_size); } -WebRtc_Word32 ViECapturer::Encode(const VideoFrame& input_image, - const CodecSpecificInfo* codec_specific_info, - const VideoFrameType frame_type) { +WebRtc_Word32 ViECapturer::Encode( + const VideoFrame& input_image, + const CodecSpecificInfo* codec_specific_info, + const std::vector* frame_types) { CriticalSectionScoped cs(encoding_cs_.get()); if (!capture_encoder_) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } - if (frame_type == kKeyFrame) { + if (frame_types == NULL) { + return capture_encoder_->EncodeFrameType(kVideoFrameDelta); + } else if ((*frame_types)[0] == kKeyFrame) { return capture_encoder_->EncodeFrameType(kVideoFrameKey); - } - if (frame_type == kSkipFrame) { + } else if ((*frame_types)[0] == kSkipFrame) { return capture_encoder_->EncodeFrameType(kFrameEmpty); } return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; diff --git a/src/video_engine/vie_capturer.h b/src/video_engine/vie_capturer.h index 6b054fe36..5fc0dad34 100644 --- a/src/video_engine/vie_capturer.h +++ b/src/video_engine/vie_capturer.h @@ -11,6 +11,8 @@ #ifndef WEBRTC_VIDEO_ENGINE_VIE_CAPTURER_H_ #define WEBRTC_VIDEO_ENGINE_VIE_CAPTURER_H_ +#include + #include "common_types.h" // NOLINT #include "engine_configurations.h" // NOLINT #include "modules/video_capture/main/interface/video_capture.h" @@ -142,7 +144,7 @@ class ViECapturer WebRtc_UWord32 max_payload_size); virtual WebRtc_Word32 Encode(const VideoFrame& input_image, const CodecSpecificInfo* codec_specific_info, - const VideoFrameType frame_type); + const std::vector* frame_types); virtual WebRtc_Word32 RegisterEncodeCompleteCallback( EncodedImageCallback* callback); virtual WebRtc_Word32 Release(); diff --git a/src/video_engine/vie_encoder.cc b/src/video_engine/vie_encoder.cc index 054ff0fc6..83fdcf541 100644 --- a/src/video_engine/vie_encoder.cc +++ b/src/video_engine/vie_encoder.cc @@ -585,7 +585,7 @@ int ViEEncoder::GetPreferedFrameSettings(int* width, int ViEEncoder::SendKeyFrame() { WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, ViEId(engine_id_, channel_id_), "%s", __FUNCTION__); - return vcm_.IntraFrameRequest(); + return vcm_.IntraFrameRequest(0); } WebRtc_Word32 ViEEncoder::SendCodecStatistics( @@ -818,7 +818,7 @@ void ViEEncoder::OnReceivedIntraFrameRequest(uint32_t /*ssrc*/) { "%s: Not not encoding new intra due to timing", __FUNCTION__); return; } - vcm_.IntraFrameRequest(); + vcm_.IntraFrameRequest(0); time_last_intra_request_ms_ = now; }