From 6062601c4d5cfe904b517f69c728f6543c08a998 Mon Sep 17 00:00:00 2001 From: Stefan Walk Date: Fri, 22 Mar 2013 13:07:02 +0100 Subject: [PATCH 1/3] Make rescaling flexible and add median filtering Previously the pyramid was done with a rescaling factor of 2 (implied by the use of pyrDown). This often leads to inferior results compared to a scale step of e.g. 0.8 (a factor of 2 is obviously faster). This commit makes the scale step configurable and uses a resonable default value. The other change in this commit is that median filtering is added. This is not described in this paper but it is done in the author's implementation. (See e.g. "Secrets of optical flow estimation and their principles", Sun et al., CVPR 2010) This serves as periodic outlier removal during optimization, leading to smoother flow fields while preserving motion edges. This includes splitting the optimization loop into two loops. --- modules/video/src/tvl1flow.cpp | 69 ++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/modules/video/src/tvl1flow.cpp b/modules/video/src/tvl1flow.cpp index d6d1ddd5d..07477bbe8 100644 --- a/modules/video/src/tvl1flow.cpp +++ b/modules/video/src/tvl1flow.cpp @@ -95,8 +95,11 @@ protected: int nscales; int warps; double epsilon; - int iterations; + int innerIterations; + int outerIterations; bool useInitialFlow; + double scaleStep; + int medianFiltering; private: void procOneScale(const Mat_& I0, const Mat_& I1, Mat_& u1, Mat_& u2); @@ -144,8 +147,11 @@ OpticalFlowDual_TVL1::OpticalFlowDual_TVL1() nscales = 5; warps = 5; epsilon = 0.01; - iterations = 300; + innerIterations = 30; + outerIterations = 10; useInitialFlow = false; + medianFiltering = 5; + scaleStep = 0.8; } void OpticalFlowDual_TVL1::calc(InputArray _I0, InputArray _I1, InputOutputArray _flow) @@ -209,8 +215,8 @@ void OpticalFlowDual_TVL1::calc(InputArray _I0, InputArray _I1, InputOutputArray // create the scales for (int s = 1; s < nscales; ++s) { - pyrDown(I0s[s - 1], I0s[s]); - pyrDown(I1s[s - 1], I1s[s]); + resize(I0s[s-1], I0s[s], Size(), scaleStep, scaleStep); + resize(I1s[s-1], I1s[s], Size(), scaleStep, scaleStep); if (I0s[s].cols < 16 || I0s[s].rows < 16) { @@ -220,11 +226,11 @@ void OpticalFlowDual_TVL1::calc(InputArray _I0, InputArray _I1, InputOutputArray if (useInitialFlow) { - pyrDown(u1s[s - 1], u1s[s]); - pyrDown(u2s[s - 1], u2s[s]); + resize(u1s[s-1], u1s[s], Size(), scaleStep, scaleStep); + resize(u2s[s-1], u2s[s], Size(), scaleStep, scaleStep); - multiply(u1s[s], Scalar::all(0.5), u1s[s]); - multiply(u2s[s], Scalar::all(0.5), u2s[s]); + multiply(u1s[s], Scalar::all(scaleStep), u1s[s]); + multiply(u2s[s], Scalar::all(scaleStep), u2s[s]); } else { @@ -256,8 +262,8 @@ void OpticalFlowDual_TVL1::calc(InputArray _I0, InputArray _I1, InputOutputArray resize(u2s[s], u2s[s - 1], I0s[s - 1].size()); // scale the optical flow with the appropriate zoom factor - multiply(u1s[s - 1], Scalar::all(2), u1s[s - 1]); - multiply(u2s[s - 1], Scalar::all(2), u2s[s - 1]); + multiply(u1s[s - 1], Scalar::all(1/scaleStep), u1s[s - 1]); + multiply(u2s[s - 1], Scalar::all(1/scaleStep), u2s[s - 1]); } Mat uxy[] = {u1s[0], u2s[0]}; @@ -853,24 +859,31 @@ void OpticalFlowDual_TVL1::procOneScale(const Mat_& I0, const Mat_ calcGradRho(I0, I1w, I1wx, I1wy, u1, u2, grad, rho_c); float error = std::numeric_limits::max(); - for (int n = 0; error > scaledEpsilon && n < iterations; ++n) + for (int n_outer = 0; error > scaledEpsilon && n_outer < outerIterations; ++n_outer) { - // estimate the values of the variable (v1, v2) (thresholding operator TH) - estimateV(I1wx, I1wy, u1, u2, grad, rho_c, v1, v2, l_t); + if (medianFiltering > 1) { + cv::medianBlur(u1, u1, medianFiltering); + cv::medianBlur(u2, u2, medianFiltering); + } + for (int n_inner = 0; error > scaledEpsilon && n_inner < innerIterations; ++n_inner) + { + // estimate the values of the variable (v1, v2) (thresholding operator TH) + estimateV(I1wx, I1wy, u1, u2, grad, rho_c, v1, v2, l_t); - // compute the divergence of the dual variable (p1, p2) - divergence(p11, p12, div_p1); - divergence(p21, p22, div_p2); + // compute the divergence of the dual variable (p1, p2) + divergence(p11, p12, div_p1); + divergence(p21, p22, div_p2); - // estimate the values of the optical flow (u1, u2) - error = estimateU(v1, v2, div_p1, div_p2, u1, u2, static_cast(theta)); + // estimate the values of the optical flow (u1, u2) + error = estimateU(v1, v2, div_p1, div_p2, u1, u2, static_cast(theta)); - // compute the gradient of the optical flow (Du1, Du2) - forwardGradient(u1, u1x, u1y); - forwardGradient(u2, u2x, u2y); + // compute the gradient of the optical flow (Du1, Du2) + forwardGradient(u1, u1x, u1y); + forwardGradient(u2, u2x, u2y); - // estimate the values of the dual variable (p1, p2) - estimateDualVariables(u1x, u1y, u2x, u2y, p11, p12, p21, p22, taut); + // estimate the values of the dual variable (p1, p2) + estimateDualVariables(u1x, u1y, u2x, u2y, p11, p12, p21, p22, taut); + } } } } @@ -923,10 +936,16 @@ CV_INIT_ALGORITHM(OpticalFlowDual_TVL1, "DenseOpticalFlow.DualTVL1", "Number of scales used to create the pyramid of images"); obj.info()->addParam(obj, "warps", obj.warps, false, 0, 0, "Number of warpings per scale"); + obj.info()->addParam(obj, "medianFiltering", obj.medianFiltering, false, 0, 0, + "Median filter kernel size (1 = no filter) (3 or 5)"); + obj.info()->addParam(obj, "scaleStep", obj.scaleStep, false, 0, 0, + "Step between scales (<1)"); obj.info()->addParam(obj, "epsilon", obj.epsilon, false, 0, 0, "Stopping criterion threshold used in the numerical scheme, which is a trade-off between precision and running time"); - obj.info()->addParam(obj, "iterations", obj.iterations, false, 0, 0, - "Stopping criterion iterations number used in the numerical scheme"); + obj.info()->addParam(obj, "innerIterations", obj.innerIterations, false, 0, 0, + "inner iterations (between outlier filtering) used in the numerical scheme"); + obj.info()->addParam(obj, "outerIterations", obj.outerIterations, false, 0, 0, + "outer iterations (number of inner loops) used in the numerical scheme"); obj.info()->addParam(obj, "useInitialFlow", obj.useInitialFlow)); } // namespace From 8d97d9598e6e413bf767cc55f8c6ae9c8cd1b0af Mon Sep 17 00:00:00 2001 From: Vladislav Vinogradov Date: Fri, 29 Mar 2013 11:36:36 +0400 Subject: [PATCH 2/3] updated GPU version of TVL1 --- modules/gpu/include/opencv2/gpu.hpp | 2 ++ modules/gpu/perf/perf_video.cpp | 3 +++ modules/gpu/src/tvl1flow.cpp | 17 +++++++++-------- modules/gpu/test/test_optflow.cpp | 7 +++++-- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/modules/gpu/include/opencv2/gpu.hpp b/modules/gpu/include/opencv2/gpu.hpp index 88703fed0..84de397dc 100644 --- a/modules/gpu/include/opencv2/gpu.hpp +++ b/modules/gpu/include/opencv2/gpu.hpp @@ -1810,6 +1810,8 @@ public: */ int iterations; + double scaleStep; + bool useInitialFlow; private: diff --git a/modules/gpu/perf/perf_video.cpp b/modules/gpu/perf/perf_video.cpp index dd39aa822..a8e828efa 100644 --- a/modules/gpu/perf/perf_video.cpp +++ b/modules/gpu/perf/perf_video.cpp @@ -434,6 +434,9 @@ PERF_TEST_P(ImagePair, Video_OpticalFlowDual_TVL1, cv::Mat flow; cv::Ptr alg = cv::createOptFlow_DualTVL1(); + alg->set("medianFiltering", 1); + alg->set("innerIterations", 1); + alg->set("outerIterations", 300); TEST_CYCLE() alg->calc(frame0, frame1, flow); diff --git a/modules/gpu/src/tvl1flow.cpp b/modules/gpu/src/tvl1flow.cpp index 49dd8caf4..df9df9fd4 100644 --- a/modules/gpu/src/tvl1flow.cpp +++ b/modules/gpu/src/tvl1flow.cpp @@ -63,6 +63,7 @@ cv::gpu::OpticalFlowDual_TVL1_GPU::OpticalFlowDual_TVL1_GPU() warps = 5; epsilon = 0.01; iterations = 300; + scaleStep = 0.8; useInitialFlow = false; } @@ -112,8 +113,8 @@ void cv::gpu::OpticalFlowDual_TVL1_GPU::operator ()(const GpuMat& I0, const GpuM // create the scales for (int s = 1; s < nscales; ++s) { - gpu::pyrDown(I0s[s - 1], I0s[s]); - gpu::pyrDown(I1s[s - 1], I1s[s]); + gpu::resize(I0s[s-1], I0s[s], Size(), scaleStep, scaleStep); + gpu::resize(I1s[s-1], I1s[s], Size(), scaleStep, scaleStep); if (I0s[s].cols < 16 || I0s[s].rows < 16) { @@ -123,11 +124,11 @@ void cv::gpu::OpticalFlowDual_TVL1_GPU::operator ()(const GpuMat& I0, const GpuM if (useInitialFlow) { - gpu::pyrDown(u1s[s - 1], u1s[s]); - gpu::pyrDown(u2s[s - 1], u2s[s]); + gpu::resize(u1s[s-1], u1s[s], Size(), scaleStep, scaleStep); + gpu::resize(u2s[s-1], u2s[s], Size(), scaleStep, scaleStep); - gpu::multiply(u1s[s], Scalar::all(0.5), u1s[s]); - gpu::multiply(u2s[s], Scalar::all(0.5), u2s[s]); + gpu::multiply(u1s[s], Scalar::all(scaleStep), u1s[s]); + gpu::multiply(u2s[s], Scalar::all(scaleStep), u2s[s]); } else { @@ -159,8 +160,8 @@ void cv::gpu::OpticalFlowDual_TVL1_GPU::operator ()(const GpuMat& I0, const GpuM gpu::resize(u2s[s], u2s[s - 1], I0s[s - 1].size()); // scale the optical flow with the appropriate zoom factor - gpu::multiply(u1s[s - 1], Scalar::all(2), u1s[s - 1]); - gpu::multiply(u2s[s - 1], Scalar::all(2), u2s[s - 1]); + gpu::multiply(u1s[s - 1], Scalar::all(1/scaleStep), u1s[s - 1]); + gpu::multiply(u2s[s - 1], Scalar::all(1/scaleStep), u2s[s - 1]); } } diff --git a/modules/gpu/test/test_optflow.cpp b/modules/gpu/test/test_optflow.cpp index 595e46cbb..ca4c462ad 100644 --- a/modules/gpu/test/test_optflow.cpp +++ b/modules/gpu/test/test_optflow.cpp @@ -435,13 +435,16 @@ GPU_TEST_P(OpticalFlowDual_TVL1, Accuracy) d_alg(loadMat(frame0, useRoi), loadMat(frame1, useRoi), d_flowx, d_flowy); cv::Ptr alg = cv::createOptFlow_DualTVL1(); + alg->set("medianFiltering", 1); + alg->set("innerIterations", 1); + alg->set("outerIterations", d_alg.iterations); cv::Mat flow; alg->calc(frame0, frame1, flow); cv::Mat gold[2]; cv::split(flow, gold); - EXPECT_MAT_SIMILAR(gold[0], d_flowx, 3e-3); - EXPECT_MAT_SIMILAR(gold[1], d_flowy, 3e-3); + EXPECT_MAT_SIMILAR(gold[0], d_flowx, 4e-3); + EXPECT_MAT_SIMILAR(gold[1], d_flowy, 4e-3); } INSTANTIATE_TEST_CASE_P(GPU_Video, OpticalFlowDual_TVL1, testing::Combine( From c89786bc2d642e081f3ea62c1b30f819e9b6cf14 Mon Sep 17 00:00:00 2001 From: Vladislav Vinogradov Date: Mon, 1 Apr 2013 10:44:27 +0400 Subject: [PATCH 3/3] fixed sanity test on Windows --- modules/video/perf/perf_tvl1optflow.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/video/perf/perf_tvl1optflow.cpp b/modules/video/perf/perf_tvl1optflow.cpp index 36f16d994..0a0f74a82 100644 --- a/modules/video/perf/perf_tvl1optflow.cpp +++ b/modules/video/perf/perf_tvl1optflow.cpp @@ -26,5 +26,5 @@ PERF_TEST_P(ImagePair, OpticalFlowDual_TVL1, testing::Values(impair("cv/optflow/ TEST_CYCLE_N(10) tvl1->calc(frame1, frame2, flow); - SANITY_CHECK(flow, 0.5); + SANITY_CHECK(flow, 0.8); }