From be4312ec3dc7c36f4799a6955f7f6e5b9786a439 Mon Sep 17 00:00:00 2001 From: alcinos Date: Wed, 27 Jan 2016 15:36:23 +0100 Subject: [PATCH 1/4] Wrap DenseOptFlow class around Farneback optical flow computation --- modules/cudaoptflow/src/farneback.cpp | 4 +- .../video/include/opencv2/video/tracking.hpp | 40 +++ modules/video/src/optflowgf.cpp | 233 ++++++++++-------- 3 files changed, 173 insertions(+), 104 deletions(-) diff --git a/modules/cudaoptflow/src/farneback.cpp b/modules/cudaoptflow/src/farneback.cpp index fb62df686..43032b4f8 100644 --- a/modules/cudaoptflow/src/farneback.cpp +++ b/modules/cudaoptflow/src/farneback.cpp @@ -93,7 +93,7 @@ namespace cv { namespace cuda { namespace device { namespace optflow_farneback namespace { - class FarnebackOpticalFlowImpl : public FarnebackOpticalFlow + class FarnebackOpticalFlowImpl : public cv::cuda::FarnebackOpticalFlow { public: FarnebackOpticalFlowImpl(int numLevels, double pyrScale, bool fastPyramids, int winSize, @@ -459,7 +459,7 @@ namespace } } -Ptr cv::cuda::FarnebackOpticalFlow::create(int numLevels, double pyrScale, bool fastPyramids, int winSize, +Ptr cv::cuda::FarnebackOpticalFlow::create(int numLevels, double pyrScale, bool fastPyramids, int winSize, int numIters, int polyN, double polySigma, int flags) { return makePtr(numLevels, pyrScale, fastPyramids, winSize, diff --git a/modules/video/include/opencv2/video/tracking.hpp b/modules/video/include/opencv2/video/tracking.hpp index d6954fecc..5afe209ce 100644 --- a/modules/video/include/opencv2/video/tracking.hpp +++ b/modules/video/include/opencv2/video/tracking.hpp @@ -508,6 +508,46 @@ public: */ CV_EXPORTS_W Ptr createOptFlow_DualTVL1(); +/** @brief Class computing a dense optical flow using the Gunnar Farneback’s algorithm. + */ +class CV_EXPORTS_W FarnebackOpticalFlow : public DenseOpticalFlow +{ +public: + virtual int getNumLevels() const = 0; + virtual void setNumLevels(int numLevels) = 0; + + virtual double getPyrScale() const = 0; + virtual void setPyrScale(double pyrScale) = 0; + + virtual bool getFastPyramids() const = 0; + virtual void setFastPyramids(bool fastPyramids) = 0; + + virtual int getWinSize() const = 0; + virtual void setWinSize(int winSize) = 0; + + virtual int getNumIters() const = 0; + virtual void setNumIters(int numIters) = 0; + + virtual int getPolyN() const = 0; + virtual void setPolyN(int polyN) = 0; + + virtual double getPolySigma() const = 0; + virtual void setPolySigma(double polySigma) = 0; + + virtual int getFlags() const = 0; + virtual void setFlags(int flags) = 0; + + static Ptr create( + int numLevels = 5, + double pyrScale = 0.5, + bool fastPyramids = false, + int winSize = 13, + int numIters = 10, + int polyN = 5, + double polySigma = 1.1, + int flags = 0); +}; + //! @} video_track } // cv diff --git a/modules/video/src/optflowgf.cpp b/modules/video/src/optflowgf.cpp index 3b61bb24b..b486edaf8 100644 --- a/modules/video/src/optflowgf.cpp +++ b/modules/video/src/optflowgf.cpp @@ -583,39 +583,63 @@ FarnebackUpdateFlow_GaussianBlur( const Mat& _R0, const Mat& _R1, } -#ifdef HAVE_OPENCL namespace cv { -class FarnebackOpticalFlow +namespace +{ +class FarnebackOpticalFlowImpl : public FarnebackOpticalFlow { public: - FarnebackOpticalFlow() + FarnebackOpticalFlowImpl(int numLevels=5, double pyrScale=0.5, bool fastPyramids=false, int winSize=13, + int numIters=10, int polyN=5, double polySigma=1.1, int flags=0) : + numLevels_(numLevels), pyrScale_(pyrScale), fastPyramids_(fastPyramids), winSize_(winSize), + numIters_(numIters), polyN_(polyN), polySigma_(polySigma), flags_(flags) { - numLevels = 5; - pyrScale = 0.5; - fastPyramids = false; - winSize = 13; - numIters = 10; - polyN = 5; - polySigma = 1.1; - flags = 0; } - int numLevels; - double pyrScale; - bool fastPyramids; - int winSize; - int numIters; - int polyN; - double polySigma; - int flags; + virtual int getNumLevels() const { return numLevels_; } + virtual void setNumLevels(int numLevels) { numLevels_ = numLevels; } + virtual double getPyrScale() const { return pyrScale_; } + virtual void setPyrScale(double pyrScale) { pyrScale_ = pyrScale; } + + virtual bool getFastPyramids() const { return fastPyramids_; } + virtual void setFastPyramids(bool fastPyramids) { fastPyramids_ = fastPyramids; } + + virtual int getWinSize() const { return winSize_; } + virtual void setWinSize(int winSize) { winSize_ = winSize; } + + virtual int getNumIters() const { return numIters_; } + virtual void setNumIters(int numIters) { numIters_ = numIters; } + + virtual int getPolyN() const { return polyN_; } + virtual void setPolyN(int polyN) { polyN_ = polyN; } + + virtual double getPolySigma() const { return polySigma_; } + virtual void setPolySigma(double polySigma) { polySigma_ = polySigma; } + + virtual int getFlags() const { return flags_; } + virtual void setFlags(int flags) { flags_ = flags; } + + virtual void calc(InputArray I0, InputArray I1, InputOutputArray flow); + +private: + int numLevels_; + double pyrScale_; + bool fastPyramids_; + int winSize_; + int numIters_; + int polyN_; + double polySigma_; + int flags_; + +#ifdef HAVE_OPENCL bool operator ()(const UMat &frame0, const UMat &frame1, UMat &flowx, UMat &flowy) { CV_Assert(frame0.channels() == 1 && frame1.channels() == 1); CV_Assert(frame0.size() == frame1.size()); - CV_Assert(polyN == 5 || polyN == 7); - CV_Assert(!fastPyramids || std::abs(pyrScale - 0.5) < 1e-6); + CV_Assert(polyN_ == 5 || polyN_ == 7); + CV_Assert(!fastPyramids_ || std::abs(pyrScale_ - 0.5) < 1e-6); const int min_size = 32; @@ -630,9 +654,9 @@ public: // Crop unnecessary levels double scale = 1; int numLevelsCropped = 0; - for (; numLevelsCropped < numLevels; numLevelsCropped++) + for (; numLevelsCropped < numLevels_; numLevelsCropped++) { - scale *= pyrScale; + scale *= pyrScale_; if (size.width*scale < min_size || size.height*scale < min_size) break; } @@ -640,7 +664,7 @@ public: frame0.convertTo(frames_[0], CV_32F); frame1.convertTo(frames_[1], CV_32F); - if (fastPyramids) + if (fastPyramids_) { // Build Gaussian pyramids using pyrDown() pyramid0_.resize(numLevelsCropped + 1); @@ -654,13 +678,13 @@ public: } } - setPolynomialExpansionConsts(polyN, polySigma); + setPolynomialExpansionConsts(polyN_, polySigma_); for (int k = numLevelsCropped; k >= 0; k--) { scale = 1; for (int i = 0; i < k; i++) - scale *= pyrScale; + scale *= pyrScale_; double sigma = (1./scale - 1) * 0.5; int smoothSize = cvRound(sigma*5) | 1; @@ -669,7 +693,7 @@ public: int width = cvRound(size.width*scale); int height = cvRound(size.height*scale); - if (fastPyramids) + if (fastPyramids_) { width = pyramid0_[k].cols; height = pyramid0_[k].rows; @@ -688,7 +712,7 @@ public: if (prevFlowX.empty()) { - if (flags & cv::OPTFLOW_USE_INITIAL_FLOW) + if (flags_ & cv::OPTFLOW_USE_INITIAL_FLOW) { resize(flowx0, curFlowX, Size(width, height), 0, 0, INTER_LINEAR); resize(flowy0, curFlowY, Size(width, height), 0, 0, INTER_LINEAR); @@ -705,8 +729,8 @@ public: { resize(prevFlowX, curFlowX, Size(width, height), 0, 0, INTER_LINEAR); resize(prevFlowY, curFlowY, Size(width, height), 0, 0, INTER_LINEAR); - multiply(1./pyrScale, curFlowX, curFlowX); - multiply(1./pyrScale, curFlowY, curFlowY); + multiply(1./pyrScale_, curFlowX, curFlowX); + multiply(1./pyrScale_, curFlowY, curFlowY); } UMat M = allocMatFromBuf(5*height, width, CV_32F, M_); @@ -717,7 +741,7 @@ public: allocMatFromBuf(5*height, width, CV_32F, R_[1]) }; - if (fastPyramids) + if (fastPyramids_) { if (!polynomialExpansionOcl(pyramid0_[k], R[0])) return false; @@ -752,18 +776,18 @@ public: if (!updateMatricesOcl(curFlowX, curFlowY, R[0], R[1], M)) return false; - if (flags & OPTFLOW_FARNEBACK_GAUSSIAN) - setGaussianBlurKernel(winSize, winSize/2*0.3f); - for (int i = 0; i < numIters; i++) + if (flags_ & OPTFLOW_FARNEBACK_GAUSSIAN) + setGaussianBlurKernel(winSize_, winSize_/2*0.3f); + for (int i = 0; i < numIters_; i++) { - if (flags & OPTFLOW_FARNEBACK_GAUSSIAN) + if (flags_ & OPTFLOW_FARNEBACK_GAUSSIAN) { - if (!updateFlow_gaussianBlur(R[0], R[1], curFlowX, curFlowY, M, bufM, winSize, i < numIters-1)) + if (!updateFlow_gaussianBlur(R[0], R[1], curFlowX, curFlowY, M, bufM, winSize_, i < numIters_-1)) return false; } else { - if (!updateFlow_boxFilter(R[0], R[1], curFlowX, curFlowY, M, bufM, winSize, i < numIters-1)) + if (!updateFlow_boxFilter(R[0], R[1], curFlowX, curFlowY, M, bufM, winSize_, i < numIters_-1)) return false; } } @@ -776,7 +800,9 @@ public: flowy = curFlowY; return true; } - + virtual void collectGarbage(){ + releaseMemory(); + } void releaseMemory() { frames_[0].release(); @@ -898,15 +924,15 @@ private: #else size_t localsize[2] = { 256, 1}; #endif - size_t globalsize[2] = { DIVUP((size_t)src.cols, localsize[0] - 2*polyN) * localsize[0], (size_t)src.rows}; + size_t globalsize[2] = { DIVUP((size_t)src.cols, localsize[0] - 2*polyN_) * localsize[0], (size_t)src.rows}; #if 0 const cv::ocl::Device &device = cv::ocl::Device::getDefault(); bool useDouble = (0 != device.doubleFPConfig()); - cv::String build_options = cv::format("-D polyN=%d -D USE_DOUBLE=%d", polyN, useDouble ? 1 : 0); + cv::String build_options = cv::format("-D polyN=%d -D USE_DOUBLE=%d", polyN_, useDouble ? 1 : 0); #else - cv::String build_options = cv::format("-D polyN=%d", polyN); + cv::String build_options = cv::format("-D polyN=%d", polyN_); #endif ocl::Kernel kernel; if (!kernel.create("polynomialExpansion", cv::ocl::video::optical_flow_farneback_oclsrc, build_options)) @@ -1036,60 +1062,43 @@ private: return false; return true; } + bool calc_ocl( InputArray _prev0, InputArray _next0, + InputOutputArray _flow0) + { + if ((5 != polyN_) && (7 != polyN_)) + return false; + if (_next0.size() != _prev0.size()) + return false; + int typePrev = _prev0.type(); + int typeNext = _next0.type(); + if ((1 != CV_MAT_CN(typePrev)) || (1 != CV_MAT_CN(typeNext))) + return false; + + std::vector flowar; + if (!_flow0.empty()) + split(_flow0, flowar); + else + { + flowar.push_back(UMat()); + flowar.push_back(UMat()); + } + if(!this->operator()(_prev0.getUMat(), _next0.getUMat(), flowar[0], flowar[1])){ + return false; + } + merge(flowar, _flow0); + return true; + } +#else // HAVE_OPENCL + virtual void collectGarbage(){} +#endif }; -static bool ocl_calcOpticalFlowFarneback( InputArray _prev0, InputArray _next0, - InputOutputArray _flow0, double pyr_scale, int levels, int winsize, - int iterations, int poly_n, double poly_sigma, int flags ) +void FarnebackOpticalFlowImpl::calc(InputArray _prev0, InputArray _next0, + InputOutputArray _flow0) { - if ((5 != poly_n) && (7 != poly_n)) - return false; - if (_next0.size() != _prev0.size()) - return false; - int typePrev = _prev0.type(); - int typeNext = _next0.type(); - if ((1 != CV_MAT_CN(typePrev)) || (1 != CV_MAT_CN(typeNext))) - return false; - - FarnebackOpticalFlow opticalFlow; - opticalFlow.numLevels = levels; - opticalFlow.pyrScale = pyr_scale; - opticalFlow.fastPyramids= false; - opticalFlow.winSize = winsize; - opticalFlow.numIters = iterations; - opticalFlow.polyN = poly_n; - opticalFlow.polySigma = poly_sigma; - opticalFlow.flags = flags; - - std::vector flowar; - if (!_flow0.empty()) - split(_flow0, flowar); - else - { - flowar.push_back(UMat()); - flowar.push_back(UMat()); - } - if (!opticalFlow(_prev0.getUMat(), _next0.getUMat(), flowar[0], flowar[1])) - return false; - merge(flowar, _flow0); - return true; -} -} -#endif // HAVE_OPENCL - -void cv::calcOpticalFlowFarneback( InputArray _prev0, InputArray _next0, - InputOutputArray _flow0, double pyr_scale, int levels, int winsize, - int iterations, int poly_n, double poly_sigma, int flags ) -{ -#ifdef HAVE_OPENCL - bool use_opencl = ocl::useOpenCL() && _flow0.isUMat(); - if( use_opencl && ocl_calcOpticalFlowFarneback(_prev0, _next0, _flow0, pyr_scale, levels, winsize, iterations, poly_n, poly_sigma, flags)) - { - CV_IMPL_ADD(CV_IMPL_OCL); - return; - } -#endif - + CV_OCL_RUN(_flow0.isUMat() && + ocl::Image2D::isFormatSupported(CV_32F, 1, false), + calc_ocl(_prev0,_next0,_flow0)) Mat prev0 = _prev0.getMat(), next0 = _next0.getMat(); const int min_size = 32; const Mat* img[2] = { &prev0, &next0 }; @@ -1097,15 +1106,16 @@ void cv::calcOpticalFlowFarneback( InputArray _prev0, InputArray _next0, int i, k; double scale; Mat prevFlow, flow, fimg; + int levels = numLevels_; CV_Assert( prev0.size() == next0.size() && prev0.channels() == next0.channels() && - prev0.channels() == 1 && pyr_scale < 1 ); + prev0.channels() == 1 && pyrScale_ < 1 ); _flow0.create( prev0.size(), CV_32FC2 ); Mat flow0 = _flow0.getMat(); for( k = 0, scale = 1; k < levels; k++ ) { - scale *= pyr_scale; + scale *= pyrScale_; if( prev0.cols*scale < min_size || prev0.rows*scale < min_size ) break; } @@ -1115,7 +1125,7 @@ void cv::calcOpticalFlowFarneback( InputArray _prev0, InputArray _next0, for( k = levels; k >= 0; k-- ) { for( i = 0, scale = 1; i < k; i++ ) - scale *= pyr_scale; + scale *= pyrScale_; double sigma = (1./scale-1)*0.5; int smooth_sz = cvRound(sigma*5)|1; @@ -1131,7 +1141,7 @@ void cv::calcOpticalFlowFarneback( InputArray _prev0, InputArray _next0, if( prevFlow.empty() ) { - if( flags & OPTFLOW_USE_INITIAL_FLOW ) + if( flags_ & OPTFLOW_USE_INITIAL_FLOW ) { resize( flow0, flow, Size(width, height), 0, 0, INTER_AREA ); flow *= scale; @@ -1142,7 +1152,7 @@ void cv::calcOpticalFlowFarneback( InputArray _prev0, InputArray _next0, else { resize( prevFlow, flow, Size(width, height), 0, 0, INTER_LINEAR ); - flow *= 1./pyr_scale; + flow *= 1./pyrScale_; } Mat R[2], I, M; @@ -1151,19 +1161,38 @@ void cv::calcOpticalFlowFarneback( InputArray _prev0, InputArray _next0, img[i]->convertTo(fimg, CV_32F); GaussianBlur(fimg, fimg, Size(smooth_sz, smooth_sz), sigma, sigma); resize( fimg, I, Size(width, height), INTER_LINEAR ); - FarnebackPolyExp( I, R[i], poly_n, poly_sigma ); + FarnebackPolyExp( I, R[i], polyN_, polySigma_ ); } FarnebackUpdateMatrices( R[0], R[1], flow, M, 0, flow.rows ); - for( i = 0; i < iterations; i++ ) + for( i = 0; i < numIters_; i++ ) { - if( flags & OPTFLOW_FARNEBACK_GAUSSIAN ) - FarnebackUpdateFlow_GaussianBlur( R[0], R[1], flow, M, winsize, i < iterations - 1 ); + if( flags_ & OPTFLOW_FARNEBACK_GAUSSIAN ) + FarnebackUpdateFlow_GaussianBlur( R[0], R[1], flow, M, winSize_, i < numIters_ - 1 ); else - FarnebackUpdateFlow_Blur( R[0], R[1], flow, M, winsize, i < iterations - 1 ); + FarnebackUpdateFlow_Blur( R[0], R[1], flow, M, winSize_, i < numIters_ - 1 ); } prevFlow = flow; } } +} // namespace +} // namespace cv + +void cv::calcOpticalFlowFarneback( InputArray _prev0, InputArray _next0, + InputOutputArray _flow0, double pyr_scale, int levels, int winsize, + int iterations, int poly_n, double poly_sigma, int flags ) +{ + Ptr optflow; + optflow = makePtr(levels,pyr_scale,false,winsize,iterations,poly_n,poly_sigma,flags); + optflow->calc(_prev0,_next0,_flow0); +} + + +cv::Ptr cv::FarnebackOpticalFlow::create(int numLevels, double pyrScale, bool fastPyramids, int winSize, + int numIters, int polyN, double polySigma, int flags) +{ + return makePtr(numLevels, pyrScale, fastPyramids, winSize, + numIters, polyN, polySigma, flags); +} From 6e3b90de9b12c02e6be93f0f3cde01769cd5078a Mon Sep 17 00:00:00 2001 From: alcinos Date: Thu, 28 Jan 2016 16:26:11 +0100 Subject: [PATCH 2/4] Add static creator for TVL1 optical flow class --- modules/core/include/opencv2/core/ptr.inl.hpp | 11 ++++++++++ .../video/include/opencv2/video/tracking.hpp | 15 +++++++++++++ modules/video/src/tvl1flow.cpp | 21 +++++++++++++++++++ modules/video/test/test_tvl1optflow.cpp | 2 +- samples/cpp/tvl1_optical_flow.cpp | 2 +- 5 files changed, 49 insertions(+), 2 deletions(-) diff --git a/modules/core/include/opencv2/core/ptr.inl.hpp b/modules/core/include/opencv2/core/ptr.inl.hpp index 3f6f214a8..8c09d93cd 100644 --- a/modules/core/include/opencv2/core/ptr.inl.hpp +++ b/modules/core/include/opencv2/core/ptr.inl.hpp @@ -358,6 +358,17 @@ Ptr makePtr(const A1& a1, const A2& a2, const A3& a3, const A4& a4, const A5& return Ptr(new T(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10)); } +template +Ptr makePtr(const A1& a1, const A2& a2, const A3& a3, const A4& a4, const A5& a5, const A6& a6, const A7& a7, const A8& a8, const A9& a9, const A10& a10, const A11& a11) +{ + return Ptr(new T(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11)); +} + +template +Ptr makePtr(const A1& a1, const A2& a2, const A3& a3, const A4& a4, const A5& a5, const A6& a6, const A7& a7, const A8& a8, const A9& a9, const A10& a10, const A11& a11, const A12& a12) +{ + return Ptr(new T(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12)); +} } // namespace cv //! @endcond diff --git a/modules/video/include/opencv2/video/tracking.hpp b/modules/video/include/opencv2/video/tracking.hpp index 5afe209ce..f0993e5f8 100644 --- a/modules/video/include/opencv2/video/tracking.hpp +++ b/modules/video/include/opencv2/video/tracking.hpp @@ -502,6 +502,21 @@ public: virtual int getMedianFiltering() const = 0; /** @copybrief getMedianFiltering @see getMedianFiltering */ virtual void setMedianFiltering(int val) = 0; + + /** @brief Creates instance of cv::DualTVL1OpticalFlow*/ + static Ptr create( + double tau = 0.25, + double lambda = 0.15, + double theta = 0.3, + int nscales = 5, + int warps = 5, + double epsilon = 0.01, + int innnerIterations = 30, + int outerIterations = 10, + double scaleStep = 0.8, + double gamma = 0.0, + int medianFiltering = 5, + bool useInitialFlow = false); }; /** @brief Creates instance of cv::DenseOpticalFlow diff --git a/modules/video/src/tvl1flow.cpp b/modules/video/src/tvl1flow.cpp index b10dc3f34..c0299e5c9 100644 --- a/modules/video/src/tvl1flow.cpp +++ b/modules/video/src/tvl1flow.cpp @@ -89,6 +89,17 @@ namespace { class OpticalFlowDual_TVL1 : public DualTVL1OpticalFlow { public: + + OpticalFlowDual_TVL1(double tau_, double lambda_, double theta_, int nscales_, int warps_, + double epsilon_, int innerIterations_, int outerIterations_, + double scaleStep_, double gamma_, int medianFiltering_, + bool useInitialFlow_) : + tau(tau_), lambda(lambda_), theta(theta_), gamma(gamma_), nscales(nscales_), + warps(warps_), epsilon(epsilon_), innerIterations(innerIterations_), + outerIterations(outerIterations_), useInitialFlow(useInitialFlow_), + scaleStep(scaleStep_), medianFiltering(medianFiltering_) + { + } OpticalFlowDual_TVL1(); void calc(InputArray I0, InputArray I1, InputOutputArray flow); @@ -1450,3 +1461,13 @@ Ptr cv::createOptFlow_DualTVL1() { return makePtr(); } + +Ptr cv::DualTVL1OpticalFlow::create( + double tau, double lambda, double theta, int nscales, int warps, + double epsilon, int innerIterations, int outerIterations, double scaleStep, + double gamma, int medianFilter, bool useInitialFlow) +{ + return makePtr(tau, lambda, theta, nscales, warps, + epsilon, innerIterations, outerIterations, + scaleStep, gamma, medianFilter, useInitialFlow); +} diff --git a/modules/video/test/test_tvl1optflow.cpp b/modules/video/test/test_tvl1optflow.cpp index e4b80637e..3976f2559 100644 --- a/modules/video/test/test_tvl1optflow.cpp +++ b/modules/video/test/test_tvl1optflow.cpp @@ -154,7 +154,7 @@ TEST(Video_calcOpticalFlowDual_TVL1, Regression) ASSERT_FALSE(frame2.empty()); Mat_ flow; - Ptr tvl1 = createOptFlow_DualTVL1(); + Ptr tvl1 = cv::DualTVL1OpticalFlow::create(); tvl1->calc(frame1, frame2, flow); diff --git a/samples/cpp/tvl1_optical_flow.cpp b/samples/cpp/tvl1_optical_flow.cpp index 55b5558eb..95871c729 100644 --- a/samples/cpp/tvl1_optical_flow.cpp +++ b/samples/cpp/tvl1_optical_flow.cpp @@ -185,7 +185,7 @@ int main(int argc, const char* argv[]) } Mat_ flow; - Ptr tvl1 = createOptFlow_DualTVL1(); + Ptr tvl1 = cv::DualTVL1OpticalFlow::create(); const double start = (double)getTickCount(); tvl1->calc(frame0, frame1, flow); From 9b70c44f009bd06ca599647a96158e08c20db1b5 Mon Sep 17 00:00:00 2001 From: alcinos Date: Thu, 28 Jan 2016 16:55:45 +0100 Subject: [PATCH 3/4] Adding interface for Sparse flow computation --- .../video/include/opencv2/video/tracking.hpp | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/modules/video/include/opencv2/video/tracking.hpp b/modules/video/include/opencv2/video/tracking.hpp index f0993e5f8..983a0c390 100644 --- a/modules/video/include/opencv2/video/tracking.hpp +++ b/modules/video/include/opencv2/video/tracking.hpp @@ -397,6 +397,27 @@ public: CV_WRAP virtual void collectGarbage() = 0; }; +/** @brief Base interface for sparse optical flow algorithms. + */ +class CV_EXPORTS_W SparseOpticalFlow : public Algorithm +{ +public: + /** @brief Calculates a sparse optical flow. + + @param prevImg First input image. + @param nextImg Second input image of the same size and the same type as prevImg. + @param prevPts Vector of 2D points for which the flow needs to be found. + @param nextPts Output vector of 2D points containing the calculated new positions of input features in the second image. + @param status Output status vector. Each element of the vector is set to 1 if the + flow for the corresponding features has been found. Otherwise, it is set to 0. + @param err Optional output vector that contains error response for each point (inverse confidence). + */ + CV_WRAP virtual void calc(InputArray prevImg, InputArray nextImg, + InputArray prevPts, InputOutputArray nextPts, + OutputArray status, + OutputArray err = cv::noArray()) = 0; +}; + /** @brief "Dual TV L1" Optical Flow Algorithm. The class implements the "Dual TV L1" optical flow algorithm described in @cite Zach2007 and @@ -563,6 +584,7 @@ public: int flags = 0); }; + //! @} video_track } // cv From e22b838af8a0078ca472583d76d0244dc9fad455 Mon Sep 17 00:00:00 2001 From: alcinos Date: Thu, 28 Jan 2016 18:45:52 +0100 Subject: [PATCH 4/4] Wrap SparseOptFlow class around PyrLK optical flow computation --- modules/cudaoptflow/src/pyrlk.cpp | 12 +- .../video/include/opencv2/video/tracking.hpp | 34 ++++++ modules/video/src/lkpyramid.cpp | 112 +++++++++++------- 3 files changed, 107 insertions(+), 51 deletions(-) diff --git a/modules/cudaoptflow/src/pyrlk.cpp b/modules/cudaoptflow/src/pyrlk.cpp index dcfd1f66d..c7f706087 100644 --- a/modules/cudaoptflow/src/pyrlk.cpp +++ b/modules/cudaoptflow/src/pyrlk.cpp @@ -47,9 +47,9 @@ using namespace cv::cuda; #if !defined (HAVE_CUDA) || defined (CUDA_DISABLER) -Ptr cv::cuda::SparsePyrLKOpticalFlow::create(Size, int, int, bool) { throw_no_cuda(); return Ptr(); } +Ptr cv::cuda::SparsePyrLKOpticalFlow::create(Size, int, int, bool) { throw_no_cuda(); return Ptr(); } -Ptr cv::cuda::DensePyrLKOpticalFlow::create(Size, int, int, bool) { throw_no_cuda(); return Ptr(); } +Ptr cv::cuda::DensePyrLKOpticalFlow::create(Size, int, int, bool) { throw_no_cuda(); return Ptr(); } #else /* !defined (HAVE_CUDA) */ @@ -283,7 +283,7 @@ namespace vPyr[idx].copyTo(v, stream); } - class SparsePyrLKOpticalFlowImpl : public SparsePyrLKOpticalFlow, private PyrLKOpticalFlowBase + class SparsePyrLKOpticalFlowImpl : public cv::cuda::SparsePyrLKOpticalFlow, private PyrLKOpticalFlowBase { public: SparsePyrLKOpticalFlowImpl(Size winSize, int maxLevel, int iters, bool useInitialFlow) : @@ -366,14 +366,14 @@ namespace }; } -Ptr cv::cuda::SparsePyrLKOpticalFlow::create(Size winSize, int maxLevel, int iters, bool useInitialFlow) +Ptr cv::cuda::SparsePyrLKOpticalFlow::create(Size winSize, int maxLevel, int iters, bool useInitialFlow) { return makePtr(winSize, maxLevel, iters, useInitialFlow); } -Ptr cv::cuda::DensePyrLKOpticalFlow::create(Size winSize, int maxLevel, int iters, bool useInitialFlow) +Ptr cv::cuda::DensePyrLKOpticalFlow::create(Size winSize, int maxLevel, int iters, bool useInitialFlow) { return makePtr(winSize, maxLevel, iters, useInitialFlow); } -#endif /* !defined (HAVE_CUDA) */ \ No newline at end of file +#endif /* !defined (HAVE_CUDA) */ diff --git a/modules/video/include/opencv2/video/tracking.hpp b/modules/video/include/opencv2/video/tracking.hpp index 983a0c390..996e60b2d 100644 --- a/modules/video/include/opencv2/video/tracking.hpp +++ b/modules/video/include/opencv2/video/tracking.hpp @@ -585,6 +585,40 @@ public: }; +/** @brief Class used for calculating a sparse optical flow. + +The class can calculate an optical flow for a sparse feature set using the +iterative Lucas-Kanade method with pyramids. + +@sa calcOpticalFlowPyrLK + +*/ +class CV_EXPORTS SparsePyrLKOpticalFlow : public SparseOpticalFlow +{ +public: + virtual Size getWinSize() const = 0; + virtual void setWinSize(Size winSize) = 0; + + virtual int getMaxLevel() const = 0; + virtual void setMaxLevel(int maxLevel) = 0; + + virtual TermCriteria getTermCriteria() const = 0; + virtual void setTermCriteria(TermCriteria& crit) = 0; + + virtual int getFlags() const = 0; + virtual void setFlags(int flags) = 0; + + virtual double getMinEigThreshold() const = 0; + virtual void setMinEigThreshold(double minEigThreshold) = 0; + + static Ptr create( + Size winSize = Size(21, 21), + int maxLevel = 3, TermCriteria crit = + TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 30, 0.01), + int flags = 0, + double minEigThreshold = 1e-4); +}; + //! @} video_track } // cv diff --git a/modules/video/src/lkpyramid.cpp b/modules/video/src/lkpyramid.cpp index 891ae7860..25614d9ce 100644 --- a/modules/video/src/lkpyramid.cpp +++ b/modules/video/src/lkpyramid.cpp @@ -837,10 +837,11 @@ int cv::buildOpticalFlowPyramid(InputArray _img, OutputArrayOfArrays pyramid, Si return maxLevel; } -#ifdef HAVE_OPENCL namespace cv { - class PyrLKOpticalFlow +namespace +{ + class SparsePyrLKOpticalFlowImpl : public SparsePyrLKOpticalFlow { struct dim3 { @@ -848,17 +849,40 @@ namespace cv dim3() : x(0), y(0), z(0) { } }; public: - PyrLKOpticalFlow() + SparsePyrLKOpticalFlowImpl(Size winSize_ = Size(21,21), + int maxLevel_ = 3, + TermCriteria criteria_ = TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 30, 0.01), + int flags_ = 0, + double minEigThreshold_ = 1e-4) : + winSize(winSize_), maxLevel(maxLevel_), criteria(criteria_), flags(flags_), minEigThreshold(minEigThreshold_) +#ifdef HAVE_OPENCL + , iters(criteria_.maxCount), derivLambda(criteria_.epsilon), useInitialFlow(0 != (flags_ & OPTFLOW_LK_GET_MIN_EIGENVALS)), waveSize(0) +#endif { - winSize = Size(21, 21); - maxLevel = 3; - iters = 30; - derivLambda = 0.5; - useInitialFlow = false; - - waveSize = 0; } + virtual Size getWinSize() const {return winSize;} + virtual void setWinSize(Size winSize_){winSize = winSize_;} + + virtual int getMaxLevel() const {return maxLevel;} + virtual void setMaxLevel(int maxLevel_){maxLevel = maxLevel_;} + + virtual TermCriteria getTermCriteria() const {return criteria;} + virtual void setTermCriteria(TermCriteria& crit_){criteria=crit_;} + + virtual int getFlags() const {return flags; } + virtual void setFlags(int flags_){flags=flags_;} + + virtual double getMinEigThreshold() const {return minEigThreshold;} + virtual void setMinEigThreshold(double minEigThreshold_){minEigThreshold=minEigThreshold_;} + + virtual void calc(InputArray prevImg, InputArray nextImg, + InputArray prevPts, InputOutputArray nextPts, + OutputArray status, + OutputArray err = cv::noArray()); + + private: +#ifdef HAVE_OPENCL bool checkParam() { iters = std::min(std::max(iters, 0), 100); @@ -930,14 +954,17 @@ namespace cv } return true; } +#endif Size winSize; int maxLevel; + TermCriteria criteria; + int flags; + double minEigThreshold; +#ifdef HAVE_OPENCL int iters; double derivLambda; bool useInitialFlow; - - private: int waveSize; bool initWaveSize() { @@ -1017,15 +1044,11 @@ namespace cv { return (cv::ocl::Device::TYPE_CPU == cv::ocl::Device::getDefault().type()); } - }; - static bool ocl_calcOpticalFlowPyrLK(InputArray _prevImg, InputArray _nextImg, - InputArray _prevPts, InputOutputArray _nextPts, - OutputArray _status, OutputArray _err, - Size winSize, int maxLevel, - TermCriteria criteria, - int flags/*, double minEigThreshold*/ ) + bool ocl_calcOpticalFlowPyrLK(InputArray _prevImg, InputArray _nextImg, + InputArray _prevPts, InputOutputArray _nextPts, + OutputArray _status, OutputArray _err) { if (0 != (OPTFLOW_LK_GET_MIN_EIGENVALS & flags)) return false; @@ -1045,7 +1068,6 @@ namespace cv if ((1 != _prevPts.size().height) && (1 != _prevPts.size().width)) return false; size_t npoints = _prevPts.total(); - bool useInitialFlow = (0 != (flags & OPTFLOW_USE_INITIAL_FLOW)); if (useInitialFlow) { if (_nextPts.empty() || _nextPts.type() != CV_32FC2 || (!_prevPts.isContinuous())) @@ -1060,14 +1082,7 @@ namespace cv _nextPts.create(_prevPts.size(), _prevPts.type()); } - PyrLKOpticalFlow opticalFlow; - opticalFlow.winSize = winSize; - opticalFlow.maxLevel = maxLevel; - opticalFlow.iters = criteria.maxCount; - opticalFlow.derivLambda = criteria.epsilon; - opticalFlow.useInitialFlow = useInitialFlow; - - if (!opticalFlow.checkParam()) + if (!checkParam()) return false; UMat umatErr; @@ -1082,28 +1097,19 @@ namespace cv _status.create((int)npoints, 1, CV_8UC1); UMat umatNextPts = _nextPts.getUMat(); UMat umatStatus = _status.getUMat(); - return opticalFlow.sparse(_prevImg.getUMat(), _nextImg.getUMat(), _prevPts.getUMat(), umatNextPts, umatStatus, umatErr); + return sparse(_prevImg.getUMat(), _nextImg.getUMat(), _prevPts.getUMat(), umatNextPts, umatStatus, umatErr); } +#endif }; -#endif -void cv::calcOpticalFlowPyrLK( InputArray _prevImg, InputArray _nextImg, +void SparsePyrLKOpticalFlowImpl::calc( InputArray _prevImg, InputArray _nextImg, InputArray _prevPts, InputOutputArray _nextPts, - OutputArray _status, OutputArray _err, - Size winSize, int maxLevel, - TermCriteria criteria, - int flags, double minEigThreshold ) + OutputArray _status, OutputArray _err) { -#ifdef HAVE_OPENCL - bool use_opencl = ocl::useOpenCL() && - (_prevImg.isUMat() || _nextImg.isUMat()) && - ocl::Image2D::isFormatSupported(CV_32F, 1, false); - if ( use_opencl && ocl_calcOpticalFlowPyrLK(_prevImg, _nextImg, _prevPts, _nextPts, _status, _err, winSize, maxLevel, criteria, flags/*, minEigThreshold*/)) - { - CV_IMPL_ADD(CV_IMPL_OCL); - return; - } -#endif + CV_OCL_RUN(ocl::useOpenCL() && + (_prevImg.isUMat() || _nextImg.isUMat()) && + ocl::Image2D::isFormatSupported(CV_32F, 1, false), + ocl_calcOpticalFlowPyrLK(_prevImg, _nextImg, _prevPts, _nextPts, _status, _err)) Mat prevPtsMat = _prevPts.getMat(); const int derivDepth = DataType::depth; @@ -1262,6 +1268,22 @@ void cv::calcOpticalFlowPyrLK( InputArray _prevImg, InputArray _nextImg, } } +} // namespace +} // namespace cv +cv::Ptr cv::SparsePyrLKOpticalFlow::create(Size winSize, int maxLevel, TermCriteria crit, int flags, double minEigThreshold){ + return makePtr(winSize,maxLevel,crit,flags,minEigThreshold); +} +void cv::calcOpticalFlowPyrLK( InputArray _prevImg, InputArray _nextImg, + InputArray _prevPts, InputOutputArray _nextPts, + OutputArray _status, OutputArray _err, + Size winSize, int maxLevel, + TermCriteria criteria, + int flags, double minEigThreshold ) +{ + Ptr optflow = cv::SparsePyrLKOpticalFlow::create(winSize,maxLevel,criteria,flags,minEigThreshold); + optflow->calc(_prevImg,_nextImg,_prevPts,_nextPts,_status,_err); +} + namespace cv {