From 21a4405d01df3a4c1c94f42ebef7b8ee042ae350 Mon Sep 17 00:00:00 2001 From: "mikhal@google.com" Date: Wed, 29 Jun 2011 17:00:03 +0000 Subject: [PATCH] VPLIB/Interpolation - Delete decode buffer only if too small, this required an API change. In addition, done some clean up and updated test and related code in VCM. Review URL: http://webrtc-codereview.appspot.com/46003 git-svn-id: http://webrtc.googlecode.com/svn/trunk@131 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../vplib/main/interface/interpolator.h | 16 +-- .../vplib/main/source/interpolator.cc | 49 +++++++-- .../vplib/main/source/scale_bilinear_yuv.cc | 99 +++++++++++-------- .../vplib/main/source/scale_bilinear_yuv.h | 15 +-- .../vplib/main/test/interpolation_test.cc | 91 +++++++++++++---- common_video/vplib/main/test/tester_main.cc | 2 +- .../main/source/spatial_resampler.cc | 35 ++++--- 7 files changed, 211 insertions(+), 96 deletions(-) diff --git a/common_video/vplib/main/interface/interpolator.h b/common_video/vplib/main/interface/interpolator.h index 70ec4fa5b..ca331d71e 100644 --- a/common_video/vplib/main/interface/interpolator.h +++ b/common_video/vplib/main/interface/interpolator.h @@ -13,8 +13,8 @@ * Interface to the WebRTC's interpolation functionality */ -#ifndef WEBRTC_COMMON_VIDEO_INTERFACE_INTERPOLATOR_H -#define WEBRTC_COMMON_VIDEO_INTERFACE_INTERPOLATOR_H +#ifndef WEBRTC_COMMON_VIDEO_VPLIB_INTERFACE_INTERPOLATOR_H +#define WEBRTC_COMMON_VIDEO_VPLIB_INTERFACE_INTERPOLATOR_H #include "typedefs.h" #include "vplib.h" @@ -49,14 +49,17 @@ public: // // Return value : Height of interpolated frame if OK, // : -1 - parameter error - // : -2 - general error + // : -2 - interpolator not set WebRtc_Word32 Interpolate(const WebRtc_UWord8* srcFrame, - WebRtc_UWord8*& dstFrame); + WebRtc_UWord8*& dstFrame, + WebRtc_UWord32& dstSize); private: // Extract computation method given actual type - WebRtc_Word32 Method(interpolatorType type); + // + // Return value : True if requested type is supported, false otherwise + bool Method(interpolatorType type); // Determine if the VideoTypes are currently supported WebRtc_Word32 SupportedVideoType(VideoType srcVideoType, @@ -67,10 +70,11 @@ private: WebRtc_UWord32 _srcHeight; WebRtc_UWord32 _dstWidth; WebRtc_UWord32 _dstHeight; + bool _set; }; } // namespace webrtc -#endif // WEBRTC_COMMON_VIDEO_INTERFACE_INTERPOLATOR_H +#endif // WEBRTC_COMMON_VIDEO_VPLIB_INTERFACE_INTERPOLATOR_H diff --git a/common_video/vplib/main/source/interpolator.cc b/common_video/vplib/main/source/interpolator.cc index 8e49eb16b..477f03404 100644 --- a/common_video/vplib/main/source/interpolator.cc +++ b/common_video/vplib/main/source/interpolator.cc @@ -21,7 +21,8 @@ _method(kBilinear), _srcWidth(0), _srcHeight(0), _dstWidth(0), -_dstHeight(0) +_dstHeight(0), +_set(false) { } @@ -36,49 +37,73 @@ interpolator::Set(WebRtc_UWord32 srcWidth, WebRtc_UWord32 srcHeight, VideoType srcVideoType, VideoType dstVideoType, interpolatorType type) { + _set = false; if (srcWidth < 1 || srcHeight < 1 || dstWidth < 1 || dstHeight < 1 ) + { return -1; + } - if (Method(type) < 0) + if (!Method(type)) + { return -1; + } if (!SupportedVideoType(srcVideoType, dstVideoType)) + { return -1; + } _srcWidth = srcWidth; _srcHeight = srcHeight; _dstWidth = dstWidth; _dstHeight = dstHeight; + _set = true; return 0; } WebRtc_Word32 interpolator::Interpolate(const WebRtc_UWord8* srcFrame, - WebRtc_UWord8*& dstFrame) + WebRtc_UWord8*& dstFrame, + WebRtc_UWord32& dstSize) { if (srcFrame == NULL) + { return -1; + } + if (!_set) + { + return -2; + } + WebRtc_Word32 ret = 0; switch (_method) { case kBilinear : - return ScaleBilinear (srcFrame, dstFrame, - _srcWidth, _srcHeight, - _dstWidth, _dstHeight); + ret = ScaleBilinear(srcFrame, dstFrame, + _srcWidth, _srcHeight, + _dstWidth, _dstHeight, + dstSize); + break; default : - return -1; + ret = -1; + break; } + return ret; } -WebRtc_Word32 +bool interpolator::Method(interpolatorType type) { + // currently only 1 supported + if (type != kBilinear) + { + return false; + } _method = type; - - return 0; + return true; } @@ -87,12 +112,16 @@ interpolator::SupportedVideoType(VideoType srcVideoType, VideoType dstVideoType) { if (srcVideoType != dstVideoType) + { return -1; + } if ((srcVideoType != kI420) || (srcVideoType != kIYUV) || (srcVideoType != kYV12)) + { return -1; + } return 0; } diff --git a/common_video/vplib/main/source/scale_bilinear_yuv.cc b/common_video/vplib/main/source/scale_bilinear_yuv.cc index b92fb73b2..dcadb1c8f 100644 --- a/common_video/vplib/main/source/scale_bilinear_yuv.cc +++ b/common_video/vplib/main/source/scale_bilinear_yuv.cc @@ -158,14 +158,11 @@ static void FilterVertical(WebRtc_UWord8* ybuf, } } - WebRtc_Word32 -ScaleBilinear(const WebRtc_UWord8* srcFrame, - WebRtc_UWord8*& dstFrame, - WebRtc_UWord32 srcWidth, - WebRtc_UWord32 srcHeight, - WebRtc_UWord32 dstWidth, - WebRtc_UWord32 dstHeight) +ScaleBilinear(const WebRtc_UWord8* srcFrame, WebRtc_UWord8*& dstFrame, + WebRtc_UWord32 srcWidth, WebRtc_UWord32 srcHeight, + WebRtc_UWord32 dstWidth, WebRtc_UWord32 dstHeight, + WebRtc_UWord32& dstSize) { // Setting source const WebRtc_UWord8* src = srcFrame; @@ -175,8 +172,8 @@ ScaleBilinear(const WebRtc_UWord8* srcFrame, const WebRtc_UWord32 srcUvStride = (((srcStride + 1 >> 1) + 15) & ~15); const WebRtc_UWord32 srcStrideArray[3] = {srcStride, - srcUvStride, - srcUvStride + srcUvStride, + srcUvStride }; const WebRtc_UWord32 srcWidthArray[3] = {srcWidth, (srcWidth + 1) >> 1, @@ -184,15 +181,16 @@ ScaleBilinear(const WebRtc_UWord8* srcFrame, }; // if srcFrame isn't aligned to nice boundaries then copy it over - // int another buffer + // in another buffer if ((srcStride > srcWidth) || (srcUvStride > ((srcWidth + 1) >> 1))) { // allocate buffer that can accommodate the stride - srcTmp = new WebRtc_UWord8[srcStride*srcHeight*3 >> 1]; + srcTmp = new WebRtc_UWord8[srcStride * srcHeight * 3 >> 1]; WebRtc_UWord8* tmpPlaneArray[3]; tmpPlaneArray[0] = srcTmp; tmpPlaneArray[1] = tmpPlaneArray[0] + srcStride * srcHeight; - tmpPlaneArray[2] = tmpPlaneArray[1] + (srcStride >> 1)*(srcHeight >> 1); + tmpPlaneArray[2] = tmpPlaneArray[1] + + (srcStride >> 1) * (srcHeight >> 1); WebRtc_UWord8* tmpPtr = srcTmp; const WebRtc_UWord8* srcPtr = srcFrame; @@ -214,38 +212,52 @@ ScaleBilinear(const WebRtc_UWord8* srcFrame, const WebRtc_UWord8* srcPlaneArray[3]; srcPlaneArray[0] = src; - srcPlaneArray[1] = srcPlaneArray[0] + srcStride*srcHeight; - srcPlaneArray[2] = srcPlaneArray[1] + (srcStride >> 1)*(srcHeight >> 1); + srcPlaneArray[1] = srcPlaneArray[0] + srcStride * srcHeight; + srcPlaneArray[2] = srcPlaneArray[1] + (srcStride >> 1) * (srcHeight >> 1); // Setting destination const WebRtc_UWord32 dstStride = (dstWidth + 31) & ~31; const WebRtc_UWord32 dstUvStride = (((dstStride + 1 >> 1) + 31) & ~31); - if (dstFrame) + WebRtc_UWord32 dstRequiredSize = dstStride * dstHeight + + 2 * (dstUvStride * ((dstHeight + 1) >> 1)); + WebRtc_UWord32 dstFinalRequiredSize = dstWidth * dstHeight * 3 >> 1; + + + if (dstFrame && dstFinalRequiredSize > dstSize) { + // allocated buffer is too small delete [] dstFrame; dstFrame = NULL; } - - WebRtc_UWord32 dstRequiredSize = dstStride*dstHeight + - 2*(dstUvStride*((dstHeight + 1) >> 1)); - dstFrame = new WebRtc_UWord8[dstRequiredSize]; if (dstFrame == NULL) - return -1; + { + dstFrame = new WebRtc_UWord8[dstFinalRequiredSize]; + dstSize = dstFinalRequiredSize; + } + WebRtc_UWord8* dstPtr = dstFrame; + WebRtc_UWord8* tmpDst = NULL; - WebRtc_UWord8* dstPlaneArray[3] = {dstFrame, - dstPlaneArray[0] + dstStride*dstHeight, + if (dstFinalRequiredSize < dstRequiredSize) + { + // allocate a tmp buffer for destination + tmpDst = new WebRtc_UWord8[dstRequiredSize]; + dstPtr = tmpDst; + } + + WebRtc_UWord8* dstPlaneArray[3] = {dstPtr, + dstPlaneArray[0] + dstStride * dstHeight, dstPlaneArray[1] + - (dstUvStride*((dstHeight + 1) >> 1)) + (dstUvStride * ((dstHeight + 1) >> 1)) }; const WebRtc_UWord32 dstStrideArray[3] = {dstStride, - dstUvStride, - dstUvStride + dstUvStride, + dstUvStride }; const WebRtc_UWord32 dstWidthArray[3] = {dstWidth, - dstWidth>>1, - dstWidth>>1 + dstWidth>>1, + dstWidth>>1 }; for (WebRtc_UWord32 p = 0; p < 3; p++) @@ -257,7 +269,7 @@ ScaleBilinear(const WebRtc_UWord8* srcFrame, WebRtc_UWord8* intermediaryBuf = new WebRtc_UWord8[srcStrideArray[p]]; const WebRtc_UWord32 hscale_fixed = (sh << kFractionBits) / dh; - const WebRtc_UWord32 source_dx = srcWidthArray[p]*kFractionMax / + const WebRtc_UWord32 source_dx = srcWidthArray[p] * kFractionMax / dstWidthArray[p]; @@ -266,18 +278,22 @@ ScaleBilinear(const WebRtc_UWord8* srcFrame, horizontalFilteredBuf = filteredBuf; if (source_dx != kFractionMax) + { horizontalFilteredBuf = intermediaryBuf; + } // horizontal filter WebRtc_UWord32 source_h_subpixel = (h * hscale_fixed); if (hscale_fixed >= (kFractionMax * 2)) + { // For 1/2 or less, center filter. source_h_subpixel += kFractionMax / 2; + } WebRtc_UWord32 source_h = source_h_subpixel >> kFractionBits; const WebRtc_UWord8* ptr_0 = srcPlaneArray[p] + - source_h*srcStrideArray[p]; + source_h * srcStrideArray[p]; const WebRtc_UWord8* ptr_1 = ptr_0 + srcStrideArray[p]; @@ -299,42 +315,47 @@ ScaleBilinear(const WebRtc_UWord8* srcFrame, // vertical filter only if necessary if (source_dx != kFractionMax) + { FilterVertical(filteredBuf, horizontalFilteredBuf, dstWidthArray[p], source_dx); + } filteredBuf += dstStrideArray[p]; } if (intermediaryBuf != NULL) + { delete [] intermediaryBuf; + } } if (srcTmp != NULL) + { delete [] srcTmp; - + } // Filtered image was placed in an aligned buffer. If the // final output is not in an aligned buffer copy the image over. if (dstStride > dstWidth) { - WebRtc_UWord8* dstFinal = - new WebRtc_UWord8[(dstWidth*dstHeight*3) >> 1]; - WebRtc_UWord8* dstPtr = dstFinal; + WebRtc_UWord8* dstFramePtr = dstFrame; for (WebRtc_UWord32 p = 0; p < 3; p++) { - WebRtc_UWord8* srcPtr = dstPlaneArray[p]; + WebRtc_UWord8* dstPlanePtr = dstPlaneArray[p]; const WebRtc_UWord32 h = (p == 0) ? dstHeight : dstHeight >> 1; for (WebRtc_UWord32 i = 0; i < h; i++) { - memcpy(dstPtr, srcPtr, dstWidthArray[p]); - dstPtr += dstWidthArray[p]; - srcPtr += dstStrideArray[p]; + memcpy(dstFramePtr, dstPlanePtr, dstWidthArray[p]); + dstFramePtr += dstWidthArray[p]; + dstPlanePtr += dstStrideArray[p]; } } + } - delete [] dstFrame; - dstFrame = dstFinal; + if (tmpDst != NULL) + { + delete [] tmpDst; } return dstHeight; diff --git a/common_video/vplib/main/source/scale_bilinear_yuv.h b/common_video/vplib/main/source/scale_bilinear_yuv.h index 08e905e23..574a3c153 100644 --- a/common_video/vplib/main/source/scale_bilinear_yuv.h +++ b/common_video/vplib/main/source/scale_bilinear_yuv.h @@ -13,8 +13,8 @@ * yuv bilinear scaler */ -#ifndef WEBRTC_COMMON_VIDEO_INTERFACE_SCALE_BILINEAR_YUV_H -#define WEBRTC_COMMON_VIDEO_INTERFACE_SCALE_BILINEAR_YUV_H +#ifndef WEBRTC_COMMON_VIDEO_VPLIB_SCALE_BILINEAR_YUV_H +#define WEBRTC_COMMON_VIDEO_VPLIB_SCALE_BILINEAR_YUV_H #include "typedefs.h" #include "vplib.h" @@ -22,10 +22,11 @@ namespace webrtc { -WebRtc_Word32 ScaleBilinear(const WebRtc_UWord8* src, WebRtc_UWord8*& dst, - WebRtc_UWord32 sW, WebRtc_UWord32 sH, - WebRtc_UWord32 dW, WebRtc_UWord32 dH); - +WebRtc_Word32 +ScaleBilinear(const WebRtc_UWord8* srcFrame, WebRtc_UWord8*& dstFrame, + WebRtc_UWord32 srcWidth, WebRtc_UWord32 srcHeight, + WebRtc_UWord32 dstWidth, WebRtc_UWord32 dstHeight, + WebRtc_UWord32& dstSize); } // namespace webrtc -#endif // WEBRTC_COMMON_VIDEO_INTERFACE_SCALE_BILINEAR_YUV_H +#endif // WEBRTC_COMMON_VIDEO_VPLIB_SCALE_BILINEAR_YUV_H diff --git a/common_video/vplib/main/test/interpolation_test.cc b/common_video/vplib/main/test/interpolation_test.cc index 9ae1ce57a..2c8ed66d0 100644 --- a/common_video/vplib/main/test/interpolation_test.cc +++ b/common_video/vplib/main/test/interpolation_test.cc @@ -18,7 +18,7 @@ using namespace webrtc; -int interpolationTest(CmdArgs& args) +int interpolation_test(CmdArgs& args) { // Read input file, interpolate first frame according to requested method // for now only YUV input and output @@ -48,7 +48,7 @@ int interpolationTest(CmdArgs& args) (webrtc::interpolatorType) args.intMethod); if (ret != 0) { - printf("Set ret = %d\n", ret); + printf("Error in set interpolator = %d\n", ret); delete inter; return ret; } @@ -57,7 +57,6 @@ int interpolationTest(CmdArgs& args) if ((outputFile = fopen(outname.c_str(), "wb")) == NULL) { printf("Cannot write file %s.\n", outname.c_str()); - exit(1); } @@ -71,21 +70,23 @@ int interpolationTest(CmdArgs& args) WebRtc_UWord32 inRequiredSize = args.width * args.height * 3 >> 1; WebRtc_UWord32 outRequiredSize = args.dstWidth * args.dstHeight * 3 >> 1; WebRtc_UWord8* inputBuffer = new WebRtc_UWord8[inRequiredSize]; - WebRtc_UWord8* outputBuffer = NULL; + WebRtc_UWord8* outputBuffer = new WebRtc_UWord8[outRequiredSize]; + WebRtc_UWord32 outputBufferSz = outRequiredSize; - // clock_t startClock, TotalClock; TotalClock = 0; - // - // running through entire sequence int frameCnt = 0; - while (feof(sourceFile)== 0) + + // running through entire sequence + while (feof(sourceFile) == 0) { if (inRequiredSize != fread(inputBuffer, 1, inRequiredSize, sourceFile)) + { break; + } startClock = clock(); - ret = inter->Interpolate(inputBuffer, outputBuffer); + ret = inter->Interpolate(inputBuffer, outputBuffer, outputBufferSz); TotalClock += clock() - startClock; if (ret == args.dstHeight) @@ -97,32 +98,86 @@ int interpolationTest(CmdArgs& args) { printf("frame #%d: Interpolation Error, ret = %d\n", frameCnt, ret); } - - if (outputBuffer) - { - delete [] outputBuffer; - outputBuffer = NULL; - } frameCnt++; printf("."); } printf("\nProcessed %d frames\n", frameCnt); if (frameCnt) + { printf("\nAvg. Time per frame[mS]: %.2lf\n", (1000.0 * static_cast(TotalClock + 0.0) /CLOCKS_PER_SEC)/frameCnt); + } + delete [] outputBuffer; + outputBuffer = NULL; + outputBufferSz = 0; + + // running some sanity checks + ret = inter->Set(0, 10, 20, 30, kI420, kI420, + (webrtc::interpolatorType) args.intMethod); + TEST(ret < 0); + ret = inter->Set(1, 10, 0, 30, kI420, kI420, + (webrtc::interpolatorType) args.intMethod); + TEST(ret < 0); + + rewind(sourceFile); + if (inRequiredSize != fread(inputBuffer, 1, inRequiredSize, sourceFile)) + { + printf("Error reading input file\n"); + return -1; + } + + ret = inter->Set(20, 10, 20, 30, kI420, kI420, + (webrtc::interpolatorType) 2); + TEST(ret < 0); + ret = inter->Interpolate(inputBuffer, outputBuffer, outputBufferSz); + TEST(ret < 0); + + // computing required size for user-defined settings + WebRtc_UWord32 dstRequiredSize = args.dstWidth * args.dstHeight * 3 >> 1; + // null output buffer - should allocate required size + ret = inter->Set(args.width, args.height, + args.dstWidth, args.dstHeight, + kI420, kI420, + (webrtc::interpolatorType) args.intMethod); + TEST(ret == 0); + ret = inter->Interpolate(inputBuffer, outputBuffer, outputBufferSz); + TEST(ret == args.dstHeight); + TEST(outputBufferSz == dstRequiredSize); if (outputBuffer) + { delete [] outputBuffer; + } + + // output buffer too small (should reallocate) + outputBufferSz = dstRequiredSize / 2; + outputBuffer = new WebRtc_UWord8[outputBufferSz]; + ret = inter->Interpolate(inputBuffer, outputBuffer, outputBufferSz); + TEST(ret == args.dstHeight); + TEST(outputBufferSz == dstRequiredSize); + if (outputBuffer) + { + delete [] outputBuffer; + } + + // output buffer too large (should maintain existing buffer size) + outputBufferSz = dstRequiredSize + 20; + outputBuffer = new WebRtc_UWord8[outputBufferSz]; + ret = inter->Interpolate(inputBuffer, outputBuffer, outputBufferSz); + TEST(ret == args.dstHeight); + TEST(outputBufferSz == dstRequiredSize + 20); + if (outputBuffer) + { + delete [] outputBuffer; + } fclose(sourceFile); fclose(outputFile); - - // wrap up delete inter; delete [] inputBuffer; - return ret; + return 0; } diff --git a/common_video/vplib/main/test/tester_main.cc b/common_video/vplib/main/test/tester_main.cc index 090a56c8f..7e0295481 100644 --- a/common_video/vplib/main/test/tester_main.cc +++ b/common_video/vplib/main/test/tester_main.cc @@ -110,7 +110,7 @@ int main(int argc, char **argv) printf("\n"); case 1: printf("VPLIB Interpolation Test\n"); - ret = interpolationTest(args); + ret = interpolation_test(args); break; case 2: printf("VPLIB Scale Test\n"); diff --git a/modules/video_processing/main/source/spatial_resampler.cc b/modules/video_processing/main/source/spatial_resampler.cc index 7d8002939..0014ef297 100644 --- a/modules/video_processing/main/source/spatial_resampler.cc +++ b/modules/video_processing/main/source/spatial_resampler.cc @@ -89,7 +89,8 @@ VPMSimpleSpatialResampler::ResampleFrame(const VideoFrame& inFrame, } // Check if re-sampling is needed - if ((inFrame.Width() == _targetWidth) && (inFrame.Height() == _targetHeight)) + if ((inFrame.Width() == _targetWidth) && + (inFrame.Height() == _targetHeight)) { return outFrame.CopyFrame(inFrame); } @@ -101,8 +102,8 @@ VPMSimpleSpatialResampler::ResampleFrame(const VideoFrame& inFrame, outFrame.SetTimeStamp(inFrame.TimeStamp()); WebRtc_UWord32 currentLength = inFrame.Width() * inFrame.Height() * 3 / 2; - if (_targetWidth > inFrame.Width() && ( ExactMultiplier(inFrame.Width(), - inFrame.Height()))) + if (_targetWidth > inFrame.Width() && + ( ExactMultiplier(inFrame.Width(), inFrame.Height()))) { // The codec might want to pad this later... adding 8 pixels const WebRtc_UWord32 requiredSize = (_targetWidth + 8) * @@ -118,7 +119,8 @@ VPMSimpleSpatialResampler::ResampleFrame(const VideoFrame& inFrame, WebRtc_UWord32 croppedHeight = inFrame.Height(); //Calculates cropped dimensions - CropSize(inFrame.Width(), inFrame.Height(), croppedWidth, croppedHeight); + CropSize(inFrame.Width(), inFrame.Height(), + croppedWidth, croppedHeight); VideoFrame* targetFrame; outFrame.VerifyAndAllocate(croppedWidth * croppedHeight * 3 / 2); @@ -308,10 +310,6 @@ VPMSimpleSpatialResampler::BiLinearInterpolation(const VideoFrame& inFrame, if (_interpolatorPtr == NULL) { _interpolatorPtr = new interpolator(); - if (_interpolatorPtr == NULL) - { - return VPM_MEMORY; - } } // set bi-linear interpolator retVal = _interpolatorPtr->Set(inFrame.Width(), inFrame.Height(), @@ -322,19 +320,26 @@ VPMSimpleSpatialResampler::BiLinearInterpolation(const VideoFrame& inFrame, return retVal; } - //Verify size of output buffer - outFrame.VerifyAndAllocate(_targetHeight * _targetWidth * 3 / 2); + // Verify size of output buffer + outFrame.VerifyAndAllocate(_targetHeight * _targetWidth * 3 >> 1); + WebRtc_UWord32 outSz = outFrame.Size(); - //interpolate frame - retVal = _interpolatorPtr->Interpolate(inFrame.Buffer(), outFrame.Buffer()); - //returns height + // interpolate frame + retVal = _interpolatorPtr->Interpolate(inFrame.Buffer(), + outFrame.Buffer(), outSz); + + assert(outSz <= outFrame.Size()); + + // returns height if (retVal < 0) + { return retVal; + } - //Set output frame parameters + // Set output frame parameters outFrame.SetHeight(_targetHeight); outFrame.SetWidth(_targetWidth); - outFrame.SetLength(_targetHeight * _targetWidth * 3 / 2); + outFrame.SetLength(outSz); outFrame.SetTimeStamp(inFrame.TimeStamp()); return VPM_OK; }