From 4e34dcbd62dc8e60927a5b439921c717c3e1b346 Mon Sep 17 00:00:00 2001 From: "marpan@webrtc.org" Date: Tue, 14 Feb 2012 17:26:24 +0000 Subject: [PATCH] Allow for spatial-downsampling without reinitializaing encoder. Change of frame size will automatically trigger new key frame in codec. This feature is set off in vie_encoder until we upgrade to the new libvpx. Also reset frame rate estimate in mediaOpt when frame rate reduction is decided. Review URL: https://webrtc-codereview.appspot.com/390006 git-svn-id: http://webrtc.googlecode.com/svn/trunk@1680 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../codecs/vp8/main/source/vp8.cc | 23 +++++-- .../main/source/media_optimization.cc | 64 ++++++++++--------- .../main/source/media_optimization.h | 2 + src/video_engine/vie_encoder.cc | 22 +------ 4 files changed, 57 insertions(+), 54 deletions(-) diff --git a/src/modules/video_coding/codecs/vp8/main/source/vp8.cc b/src/modules/video_coding/codecs/vp8/main/source/vp8.cc index 9a0141762..fc5249b8a 100644 --- a/src/modules/video_coding/codecs/vp8/main/source/vp8.cc +++ b/src/modules/video_coding/codecs/vp8/main/source/vp8.cc @@ -324,11 +324,24 @@ int VP8Encoder::Encode(const RawImage& input_image, if (encoded_complete_callback_ == NULL) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } - // image in vpx_image_t format - raw_->planes[PLANE_Y] = input_image._buffer; - raw_->planes[PLANE_U] = &input_image._buffer[codec_.height * codec_.width]; - raw_->planes[PLANE_V] = - &input_image._buffer[codec_.height * codec_.width * 5 >> 2]; + + // Check for change in frame size. + if (input_image._width != codec_.width || + input_image._height != codec_.height) { + codec_.width = input_image._width; + codec_.height = input_image._height; + + // Update encoder context for new frame size. + // Change of frame size will automatically trigger a key frame. + config_->g_w = codec_.width; + config_->g_h = codec_.height; + if (vpx_codec_enc_config_set(encoder_, config_)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + } + + vpx_img_wrap(raw_, IMG_FMT_I420, codec_.width, codec_.height, 1, + input_image._buffer); int flags = 0; #if WEBRTC_LIBVPX_VERSION >= 971 diff --git a/src/modules/video_coding/main/source/media_optimization.cc b/src/modules/video_coding/main/source/media_optimization.cc index e2c432980..4053d8ded 100644 --- a/src/modules/video_coding/main/source/media_optimization.cc +++ b/src/modules/video_coding/main/source/media_optimization.cc @@ -24,6 +24,8 @@ _maxBitRate(0), _sendCodecType(kVideoCodecUnknown), _codecWidth(0), _codecHeight(0), +_initCodecWidth(0), +_initCodecHeight(0), _userFrameRate(0), _packetLossEnc(0), _fractionLost(0), @@ -64,7 +66,7 @@ WebRtc_Word32 VCMMediaOptimization::Reset() { memset(_incomingFrameTimes, -1, sizeof(_incomingFrameTimes)); - InputFrameRate(); // Resets _incomingFrameRate + _incomingFrameRate = 0.0; _frameDropper->Reset(); _lossProtLogic->Reset(_clock->MillisecondTimestamp()); _frameDropper->SetRates(0, 0); @@ -280,6 +282,8 @@ VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, _frameDropper->SetRates(static_cast(bitRate), static_cast(frameRate)); _userFrameRate = static_cast(frameRate); + _initCodecWidth = width; + _initCodecHeight = height; _codecWidth = width; _codecHeight = height; _numLayers = (numLayers <= 1) ? 1 : numLayers; // Can also be zero. @@ -575,43 +579,51 @@ VCMMediaOptimization::QMUpdate(VCMResolutionScale* qm) // Check for no change if (qm->spatialHeightFact == 1 && qm->spatialWidthFact == 1 && - qm->temporalFact == 1) - { - return false; + qm->temporalFact == 1) { + return false; } - // Content metrics hold native values - VideoContentMetrics* cm = _content->LongTermAvgData(); - // Temporal WebRtc_UWord32 frameRate = static_cast (_incomingFrameRate + 0.5f); // Check if go back up in temporal resolution - if (qm->temporalFact == 0) - { - frameRate = (WebRtc_UWord32) 2 * _incomingFrameRate; + if (qm->temporalFact == 0) { + // Currently only allow for 1/2 frame rate reduction per action. + // TODO (marpan): allow for 2/3 reduction + frameRate = (WebRtc_UWord32) 2 * _incomingFrameRate; } // go down in temporal resolution - else - { - frameRate = (WebRtc_UWord32)(_incomingFrameRate / qm->temporalFact + 1); + else { + frameRate = (WebRtc_UWord32)(_incomingFrameRate / qm->temporalFact + 1); + } + // Reset _incomingFrameRate if temporal action was selected. + if (qm->temporalFact != 1) { + memset(_incomingFrameTimes, -1, sizeof(_incomingFrameTimes)); + _incomingFrameRate = frameRate; } // Spatial - WebRtc_UWord32 height = _codecHeight; WebRtc_UWord32 width = _codecWidth; - // Check if go back up in spatial resolution - if (qm->spatialHeightFact == 0 && qm->spatialWidthFact == 0) - { - height = cm->nativeHeight; - width = cm->nativeWidth; + WebRtc_UWord32 height = _codecHeight; + // Check if go back up in spatial resolution, and update frame sizes. + // Currently only allow for 2x2 spatial down-sampling. + // TODO (marpan): allow for 1x2, 2x1, and 4/3x4/3 (or 3/2x3/2). + if (qm->spatialHeightFact == 0 && qm->spatialWidthFact == 0) { + width = _codecWidth * 2; + height = _codecHeight * 2; } - else - { - height = _codecHeight / qm->spatialHeightFact; - width = _codecWidth / qm->spatialWidthFact; + else { + width = _codecWidth / qm->spatialWidthFact; + height = _codecHeight / qm->spatialHeightFact; } + _codecWidth = width; + _codecHeight = height; + + // New frame sizes should never exceed the original sizes + // from SetEncodingData(). + assert(_codecWidth <= _initCodecWidth); + assert(_codecHeight <= _initCodecHeight); WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, _id, "Quality Mode Update: W = %d, H = %d, FR = %f", @@ -623,8 +635,6 @@ VCMMediaOptimization::QMUpdate(VCMResolutionScale* qm) return true; } - - void VCMMediaOptimization::UpdateIncomingFrameRate() { @@ -671,10 +681,6 @@ VCMMediaOptimization::ProcessIncomingFrameRate(WebRtc_Word64 now) _incomingFrameRate = nrOfFrames * 1000.0f / static_cast(diff); } } - else - { - _incomingFrameRate = static_cast(nrOfFrames); - } } WebRtc_UWord32 diff --git a/src/modules/video_coding/main/source/media_optimization.h b/src/modules/video_coding/main/source/media_optimization.h index 7d87a6d04..14e5d1a2d 100644 --- a/src/modules/video_coding/main/source/media_optimization.h +++ b/src/modules/video_coding/main/source/media_optimization.h @@ -168,6 +168,8 @@ private: VideoCodecType _sendCodecType; WebRtc_UWord16 _codecWidth; WebRtc_UWord16 _codecHeight; + WebRtc_UWord16 _initCodecWidth; + WebRtc_UWord16 _initCodecHeight; float _userFrameRate; VCMFrameDropper* _frameDropper; diff --git a/src/video_engine/vie_encoder.cc b/src/video_engine/vie_encoder.cc index c80b5be74..646449755 100644 --- a/src/video_engine/vie_encoder.cc +++ b/src/video_engine/vie_encoder.cc @@ -823,26 +823,8 @@ WebRtc_Word32 QMTestVideoSettingsCallback::SetVideoQMSettings( const WebRtc_UWord32 frame_rate, const WebRtc_UWord32 width, const WebRtc_UWord32 height) { - WebRtc_Word32 ret_val = 0; - ret_val = vpm_->SetTargetResolution(width, height, frame_rate); - - if (!ret_val) { - // Get current settings. - VideoCodec current_codec; - vcm_->SendCodec(¤t_codec); - WebRtc_UWord32 current_bit_rate = vcm_->Bitrate(); - - // Set the new calues. - current_codec.height = static_cast(height); - current_codec.width = static_cast(width); - current_codec.maxFramerate = static_cast(frame_rate); - current_codec.startBitrate = current_bit_rate; - - // Re-register encoder with the updated settings. - ret_val = vcm_->RegisterSendCodec(¤t_codec, num_cores_, - max_payload_length_); - } - return ret_val; + // Update VPM with encoder target resolution + return vpm_->SetTargetResolution(width, height, frame_rate); } void QMTestVideoSettingsCallback::SetMaxPayloadLength(