Refactored videostab module, added base class for woobble suppression

modules/videostab/include/opencv2/videostab/global_motion.hpp

@@ -105,6 +105,15 @@ protected:
     MotionModel motionModel_;
 };
 
+class CV_EXPORTS EyeMotionEstimator : public GlobalMotionEstimatorBase
+{
+public:
+    virtual Mat estimate(const Mat &/*frame0*/, const Mat &/*frame1*/)
+    {
+        return Mat::eye(3, 3, CV_32F);
+    }
+};
+
 class CV_EXPORTS PyrLkRobustMotionEstimator : public GlobalMotionEstimatorBase
 {
 public:

modules/videostab/include/opencv2/videostab/stabilizer.hpp

@@ -52,6 +52,7 @@
 #include "opencv2/videostab/log.hpp"
 #include "opencv2/videostab/inpainting.hpp"
 #include "opencv2/videostab/deblurring.hpp"
+#include "opencv2/videostab/wobble_suppression.hpp"
 
 namespace cv
 {
@@ -93,14 +94,14 @@ public:
 protected:
     StabilizerBase();
 
-    void setUp(int cacheSize, const Mat &frame);
+    void reset();
     Mat nextStabilizedFrame();
     bool doOneIteration();
-    void stabilizeFrame(const Mat &stabilizationMotion);
+    virtual void setUp(const Mat &firstFrame);
-
+    virtual Mat estimateMotion() = 0;
-    virtual void setUp(Mat &firstFrame) = 0;
+    virtual Mat estimateStabilizationMotion() = 0;
-    virtual void stabilizeFrame() = 0;
+    void stabilizeFrame();
-    virtual void estimateMotion() = 0;
+    virtual Mat postProcessFrame(const Mat &frame);
 
     Ptr<ILog> log_;
     Ptr<IFrameSource> frameSource_;
@@ -120,6 +121,7 @@ protected:
     Mat preProcessedFrame_;
     bool doInpainting_;
     Mat inpaintingMask_;
+    Mat finalFrame_;
     std::vector<Mat> frames_;
     std::vector<Mat> motions_; // motions_[i] is the motion from i-th to i+1-th frame
     std::vector<float> blurrinessRates_;
@@ -140,9 +142,10 @@ public:
     virtual Mat nextFrame() { return nextStabilizedFrame(); }
 
 private:
-    virtual void setUp(Mat &firstFrame);
+    virtual void setUp(const Mat &firstFrame);
-    virtual void estimateMotion();
+    virtual Mat estimateMotion();
-    virtual void stabilizeFrame();
+    virtual Mat estimateStabilizationMotion();
+    virtual Mat postProcessFrame(const Mat &frame);
 
     Ptr<MotionFilterBase> motionFilter_;
 };
@@ -153,7 +156,10 @@ public:
     TwoPassStabilizer();
 
     void setMotionStabilizer(Ptr<IMotionStabilizer> val) { motionStabilizer_ = val; }
-    Ptr<IMotionStabilizer> motionStabilizer() const { return motionStabilizer_; }
+    Ptr<IMotionStabilizer> motionStabilizer() const { return motionStabilizer_; }    
+
+    void setWobbleSuppressor(Ptr<WobbleSuppressorBase> val) { wobbleSuppressor_ = val; }
+    Ptr<WobbleSuppressorBase> wobbleSuppressor() const { return wobbleSuppressor_; }
 
     void setEstimateTrimRatio(bool val) { mustEstTrimRatio_ = val; }
     bool mustEstimateTrimaRatio() const { return mustEstTrimRatio_; }
@@ -165,18 +171,20 @@ public:
     std::vector<Mat> motions() const;
 
 private:
-    void resetImpl();
     void runPrePassIfNecessary();
 
-    virtual void setUp(Mat &firstFrame);
+    virtual void setUp(const Mat &firstFrame);
-    virtual void estimateMotion() { /* do nothing as motion was estimation in pre-pass */ }
+    virtual Mat estimateMotion();
-    virtual void stabilizeFrame();
+    virtual Mat estimateStabilizationMotion();
+    virtual Mat postProcessFrame(const Mat &frame);
 
     Ptr<IMotionStabilizer> motionStabilizer_;
+    Ptr<WobbleSuppressorBase> wobbleSuppressor_;
     bool mustEstTrimRatio_;
 
     int frameCount_;
     bool isPrePassDone_;
+    Mat suppressedFrame_;
 };
 
 } // namespace videostab

modules/videostab/include/opencv2/videostab/wobble_suppression.hpp

@@ -0,0 +1,91 @@
+/*M///////////////////////////////////////////////////////////////////////////////////////
+//
+//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+//
+//  By downloading, copying, installing or using the software you agree to this license.
+//  If you do not agree to this license, do not download, install,
+//  copy or use the software.
+//
+//
+//                           License Agreement
+//                For Open Source Computer Vision Library
+//
+// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
+// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved.
+// Third party copyrights are property of their respective owners.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+//   * Redistribution's of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//
+//   * Redistribution's in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//
+//   * The name of the copyright holders may not be used to endorse or promote products
+//     derived from this software without specific prior written permission.
+//
+// This software is provided by the copyright holders and contributors "as is" and
+// any express or implied warranties, including, but not limited to, the implied
+// warranties of merchantability and fitness for a particular purpose are disclaimed.
+// In no event shall the Intel Corporation or contributors be liable for any direct,
+// indirect, incidental, special, exemplary, or consequential damages
+// (including, but not limited to, procurement of substitute goods or services;
+// loss of use, data, or profits; or business interruption) however caused
+// and on any theory of liability, whether in contract, strict liability,
+// or tort (including negligence or otherwise) arising in any way out of
+// the use of this software, even if advised of the possibility of such damage.
+//
+//M*/
+
+#ifndef __OPENCV_VIDEOSTAB_WOBBLE_SUPPRESSION_HPP__
+#define __OPENCV_VIDEOSTAB_WOBBLE_SUPPRESSION_HPP__
+
+#include <vector>
+#include "opencv2/core/core.hpp"
+#include "opencv2/videostab/global_motion.hpp"
+
+namespace cv
+{
+namespace videostab
+{
+
+class CV_EXPORTS WobbleSuppressorBase
+{
+public:
+    virtual ~WobbleSuppressorBase() {}
+
+    virtual void setFrames(const std::vector<Mat> &val) { frames_ = &val; }
+    virtual const std::vector<Mat>& frames() const { return *frames_; }
+
+    virtual void setMotions(const std::vector<Mat> &val) { motions_ = &val; }
+    virtual const std::vector<Mat>& motions() const { return *motions_; }
+
+    virtual void setStabilizedFrames(const std::vector<Mat> &val) { stabilizedFrames_ = &val; }
+    virtual const std::vector<Mat>& stabilizedFrames() const { return *stabilizedFrames_; }
+
+    virtual void setStabilizationMotions(const std::vector<Mat> &val) { stabilizationMotions_ = &val; }
+    virtual const std::vector<Mat>& stabilizationMotions() const { return *stabilizationMotions_; }
+
+    virtual void suppress(int idx, Mat &frame) = 0;
+
+protected:
+    const std::vector<Mat> *frames_;
+    const std::vector<Mat> *motions_;
+    const std::vector<Mat> *stabilizedFrames_;
+    const std::vector<Mat> *stabilizationMotions_;
+};
+
+class CV_EXPORTS NullWobbleSuppressor : public WobbleSuppressorBase
+{
+public:
+    virtual void suppress(int idx, Mat &result);
+};
+
+} // namespace videostab
+} // namespace cv
+
+#endif
+

modules/videostab/src/stabilizer.cpp

@@ -65,52 +65,40 @@ StabilizerBase::StabilizerBase()
 }
 
 
-void StabilizerBase::setUp(int cacheSize, const Mat &frame)
+void StabilizerBase::reset()
 {
-    InpainterBase *inpainter = static_cast<InpainterBase*>(inpainter_);
+    frameSize_ = Size(0, 0);
-    doInpainting_ = dynamic_cast<NullInpainter*>(inpainter) == 0;
+    frameMask_ = Mat();
-    if (doInpainting_)
+    curPos_ = -1;
-    {
+    curStabilizedPos_ = -1;
-        inpainter_->setMotionModel(motionEstimator_->motionModel());
+    doDeblurring_ = false;
-        inpainter_->setFrames(frames_);
+    preProcessedFrame_ = Mat();
-        inpainter_->setMotions(motions_);
+    doInpainting_ = false;
-        inpainter_->setStabilizedFrames(stabilizedFrames_);
+    inpaintingMask_ = Mat();
-        inpainter_->setStabilizationMotions(stabilizationMotions_);
+    frames_.clear();
-    }
+    motions_.clear();
-
+    blurrinessRates_.clear();
-    DeblurerBase *deblurer = static_cast<DeblurerBase*>(deblurer_);
+    stabilizedFrames_.clear();
-    doDeblurring_ = dynamic_cast<NullDeblurer*>(deblurer) == 0;
+    stabilizedMasks_.clear();
-    if (doDeblurring_)
+    stabilizationMotions_.clear();
-    {
-        blurrinessRates_.resize(cacheSize);
-        float blurriness = calcBlurriness(frame);
-        for (int i  = -radius_; i <= 0; ++i)
-            at(i, blurrinessRates_) = blurriness;
-        deblurer_->setFrames(frames_);
-        deblurer_->setMotions(motions_);
-        deblurer_->setBlurrinessRates(blurrinessRates_);
-    }
-
-    log_->print("processing frames");
 }
 
 
 Mat StabilizerBase::nextStabilizedFrame()
 {
+    // check if we've processed all frames already
     if (curStabilizedPos_ == curPos_ && curStabilizedPos_ != -1)
-        return Mat(); // we've processed all frames already
+        return Mat();
 
     bool processed;
     do processed = doOneIteration();
     while (processed && curStabilizedPos_ == -1);
 
+    // check if frame source is empty
     if (curStabilizedPos_ == -1)
-        return Mat(); // frame source is empty
+        return Mat();
 
-    const Mat &stabilizedFrame = at(curStabilizedPos_, stabilizedFrames_);
+    return postProcessFrame(at(curStabilizedPos_, stabilizedFrames_));
-    int dx = static_cast<int>(floor(trimRatio_ * stabilizedFrame.cols));
-    int dy = static_cast<int>(floor(trimRatio_ * stabilizedFrame.rows));
-    return stabilizedFrame(Rect(dx, dy, stabilizedFrame.cols - 2*dx, stabilizedFrame.rows - 2*dy));
 }
 
 
@@ -128,7 +116,7 @@ bool StabilizerBase::doOneIteration()
             if (doDeblurring_)
                 at(curPos_, blurrinessRates_) = calcBlurriness(frame);
 
-            estimateMotion();
+            at(curPos_ - 1, motions_) = estimateMotion();
 
             if (curPos_ >= radius_)
             {
@@ -158,15 +146,43 @@ bool StabilizerBase::doOneIteration()
 }
 
 
-void StabilizerBase::stabilizeFrame(const Mat &stabilizationMotion)
+void StabilizerBase::setUp(const Mat &firstFrame)
 {
-    Mat stabilizationMotion_;
+    InpainterBase *inpainter = static_cast<InpainterBase*>(inpainter_);
-    if (doCorrectionForInclusion_)
+    doInpainting_ = dynamic_cast<NullInpainter*>(inpainter) == 0;
-        stabilizationMotion_ = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_);
+    if (doInpainting_)
-    else
+    {
-        stabilizationMotion_ = stabilizationMotion.clone();
+        inpainter_->setMotionModel(motionEstimator_->motionModel());
+        inpainter_->setFrames(frames_);
+        inpainter_->setMotions(motions_);
+        inpainter_->setStabilizedFrames(stabilizedFrames_);
+        inpainter_->setStabilizationMotions(stabilizationMotions_);
+    }
 
-    at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion_;
+    DeblurerBase *deblurer = static_cast<DeblurerBase*>(deblurer_);
+    doDeblurring_ = dynamic_cast<NullDeblurer*>(deblurer) == 0;
+    if (doDeblurring_)
+    {
+        blurrinessRates_.resize(2*radius_ + 1);
+        float blurriness = calcBlurriness(firstFrame);
+        for (int i  = -radius_; i <= 0; ++i)
+            at(i, blurrinessRates_) = blurriness;
+        deblurer_->setFrames(frames_);
+        deblurer_->setMotions(motions_);
+        deblurer_->setBlurrinessRates(blurrinessRates_);
+    }
+
+    log_->print("processing frames");
+}
+
+
+void StabilizerBase::stabilizeFrame()
+{
+    Mat stabilizationMotion = estimateStabilizationMotion();
+    if (doCorrectionForInclusion_)
+        stabilizationMotion = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_);
+
+    at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion;
 
     if (doDeblurring_)
     {
@@ -181,22 +197,22 @@ void StabilizerBase::stabilizeFrame(const Mat &stabilizationMotion)
     if (motionEstimator_->motionModel() != HOMOGRAPHY)
         warpAffine(
                 preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_),
-                stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_LINEAR, borderMode_);
+                stabilizationMotion(Rect(0,0,3,2)), frameSize_, INTER_LINEAR, borderMode_);
     else
         warpPerspective(
                 preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_),
-                stabilizationMotion_, frameSize_, INTER_LINEAR, borderMode_);
+                stabilizationMotion, frameSize_, INTER_LINEAR, borderMode_);
 
     if (doInpainting_)
     {
         if (motionEstimator_->motionModel() != HOMOGRAPHY)
             warpAffine(
                     frameMask_, at(curStabilizedPos_, stabilizedMasks_),
-                    stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_NEAREST);
+                    stabilizationMotion(Rect(0,0,3,2)), frameSize_, INTER_NEAREST);
         else
             warpPerspective(
                     frameMask_, at(curStabilizedPos_, stabilizedMasks_),
-                    stabilizationMotion_, frameSize_, INTER_NEAREST);
+                    stabilizationMotion, frameSize_, INTER_NEAREST);
 
         erode(at(curStabilizedPos_, stabilizedMasks_), at(curStabilizedPos_, stabilizedMasks_),
               Mat());
@@ -209,6 +225,15 @@ void StabilizerBase::stabilizeFrame(const Mat &stabilizationMotion)
 }
 
 
+Mat StabilizerBase::postProcessFrame(const Mat &frame)
+{
+    // trim frame
+    int dx = static_cast<int>(floor(trimRatio_ * frame.cols));
+    int dy = static_cast<int>(floor(trimRatio_ * frame.rows));
+    return frame(Rect(dx, dy, frame.cols - 2*dx, frame.rows - 2*dy));
+}
+
+
 OnePassStabilizer::OnePassStabilizer()
 {
     setMotionFilter(new GaussianMotionFilter());
@@ -218,25 +243,17 @@ OnePassStabilizer::OnePassStabilizer()
 
 void OnePassStabilizer::reset()
 {
-    curPos_ = -1;
+    StabilizerBase::reset();
-    curStabilizedPos_ = -1;
-    frames_.clear();
-    motions_.clear();
-    stabilizedFrames_.clear();
-    stabilizationMotions_.clear();
-    doDeblurring_ = false;
-    doInpainting_ = false;    
 }
 
 
-void OnePassStabilizer::setUp(Mat &firstFrame)
+void OnePassStabilizer::setUp(const Mat &firstFrame)
 {
     frameSize_ = firstFrame.size();
     frameMask_.create(frameSize_, CV_8U);
     frameMask_.setTo(255);
 
     int cacheSize = 2*radius_ + 1;
-
     frames_.resize(cacheSize);
     stabilizedFrames_.resize(cacheSize);
     stabilizedMasks_.resize(cacheSize);
@@ -251,27 +268,32 @@ void OnePassStabilizer::setUp(Mat &firstFrame)
 
     at(0, frames_) = firstFrame;
 
-    StabilizerBase::setUp(cacheSize, firstFrame);
+    StabilizerBase::setUp(firstFrame);
 }
 
 
-void OnePassStabilizer::estimateMotion()
+Mat OnePassStabilizer::estimateMotion()
 {
-    at(curPos_ - 1, motions_) = motionEstimator_->estimate(
+    return motionEstimator_->estimate(at(curPos_ - 1, frames_), at(curPos_, frames_));
-            at(curPos_ - 1, frames_), at(curPos_, frames_));
 }
 
 
-void OnePassStabilizer::stabilizeFrame()
+Mat OnePassStabilizer::estimateStabilizationMotion()
 {
-    Mat stabilizationMotion = motionFilter_->stabilize(curStabilizedPos_, motions_, make_pair(0, curPos_));
+    return motionFilter_->stabilize(curStabilizedPos_, motions_, make_pair(0, curPos_));
-    StabilizerBase::stabilizeFrame(stabilizationMotion);
+}
+
+
+Mat OnePassStabilizer::postProcessFrame(const Mat &frame)
+{
+    return StabilizerBase::postProcessFrame(frame);
 }
 
 
 TwoPassStabilizer::TwoPassStabilizer()
 {
     setMotionStabilizer(new GaussianMotionFilter());
+    setWobbleSuppressor(new NullWobbleSuppressor());
     setEstimateTrimRatio(false);
     reset();
 }
@@ -279,16 +301,10 @@ TwoPassStabilizer::TwoPassStabilizer()
 
 void TwoPassStabilizer::reset()
 {
-    isPrePassDone_ = false;
+    StabilizerBase::reset();
     frameCount_ = 0;
-    curPos_ = -1;
+    isPrePassDone_ = false;
-    curStabilizedPos_ = -1;
+    suppressedFrame_ = Mat();
-    frames_.clear();
-    motions_.clear();
-    stabilizedFrames_.clear();
-    stabilizationMotions_.clear();
-    doDeblurring_ = false;
-    doInpainting_ = false;
 }
 
 
@@ -360,10 +376,9 @@ void TwoPassStabilizer::runPrePassIfNecessary()
 }
 
 
-void TwoPassStabilizer::setUp(Mat &firstFrame)
+void TwoPassStabilizer::setUp(const Mat &firstFrame)
 {
     int cacheSize = 2*radius_ + 1;
-
     frames_.resize(cacheSize);
     stabilizedFrames_.resize(cacheSize);
     stabilizedMasks_.resize(cacheSize);
@@ -371,13 +386,31 @@ void TwoPassStabilizer::setUp(Mat &firstFrame)
     for (int i = -radius_; i <= 0; ++i)
         at(i, frames_) = firstFrame;
 
-    StabilizerBase::setUp(cacheSize, firstFrame);
+    wobbleSuppressor_->setFrames(frames_);
+    wobbleSuppressor_->setMotions(motions_);
+    wobbleSuppressor_->setStabilizedFrames(stabilizedFrames_);
+    wobbleSuppressor_->setStabilizationMotions(stabilizationMotions_);
+
+    StabilizerBase::setUp(firstFrame);
 }
 
 
-void TwoPassStabilizer::stabilizeFrame()
+Mat TwoPassStabilizer::estimateMotion()
 {
-    StabilizerBase::stabilizeFrame(stabilizationMotions_[curStabilizedPos_]);
+    return motions_[curPos_ - 1].clone();
+}
+
+
+Mat TwoPassStabilizer::estimateStabilizationMotion()
+{
+    return stabilizationMotions_[curStabilizedPos_].clone();
+}
+
+
+Mat TwoPassStabilizer::postProcessFrame(const Mat &/*frame*/)
+{
+    wobbleSuppressor_->suppress(curStabilizedPos_, suppressedFrame_);
+    return StabilizerBase::postProcessFrame(suppressedFrame_);
 }
 
 } // namespace videostab

modules/videostab/src/wobble_suppression.cpp

@@ -0,0 +1,60 @@
+/*M///////////////////////////////////////////////////////////////////////////////////////
+//
+//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+//
+//  By downloading, copying, installing or using the software you agree to this license.
+//  If you do not agree to this license, do not download, install,
+//  copy or use the software.
+//
+//
+//                           License Agreement
+//                For Open Source Computer Vision Library
+//
+// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
+// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved.
+// Third party copyrights are property of their respective owners.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+//   * Redistribution's of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//
+//   * Redistribution's in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//
+//   * The name of the copyright holders may not be used to endorse or promote products
+//     derived from this software without specific prior written permission.
+//
+// This software is provided by the copyright holders and contributors "as is" and
+// any express or implied warranties, including, but not limited to, the implied
+// warranties of merchantability and fitness for a particular purpose are disclaimed.
+// In no event shall the Intel Corporation or contributors be liable for any direct,
+// indirect, incidental, special, exemplary, or consequential damages
+// (including, but not limited to, procurement of substitute goods or services;
+// loss of use, data, or profits; or business interruption) however caused
+// and on any theory of liability, whether in contract, strict liability,
+// or tort (including negligence or otherwise) arising in any way out of
+// the use of this software, even if advised of the possibility of such damage.
+//
+//M*/
+
+#include "precomp.hpp"
+#include "opencv2/videostab/wobble_suppression.hpp"
+#include "opencv2/videostab/ring_buffer.hpp"
+
+using namespace std;
+
+namespace cv
+{
+namespace videostab
+{
+
+void NullWobbleSuppressor::suppress(int idx, Mat &result)
+{
+    result = at(idx, *stabilizedFrames_);
+}
+
+} // namespace videostab
+} // namespace cv

samples/cpp/videostab.cpp

@@ -71,9 +71,11 @@ void run()
 {
     VideoWriter writer;
     Mat stabilizedFrame;
+    int nframes = 0;
 
     while (!(stabilizedFrame = stabilizedFrames->nextFrame()).empty())
     {
+        nframes++;
         if (!saveMotionsPath.empty())
             saveMotionsIfNecessary();
         if (!outputPath.empty())
@@ -91,7 +93,9 @@ void run()
         }
     }
 
-    cout << "\nfinished\n";
+    cout << endl
+         << "processed frames: " << nframes << endl
+         << "finished\n";
 }
 
 
@@ -248,7 +252,7 @@ int main(int argc, const char **argv)
         if (inputPath.empty()) throw runtime_error("specify video file path");
 
         VideoFileSource *source = new VideoFileSource(inputPath);
-        cout << "frame count: " << source->count() << endl;
+        cout << "frame count (rough): " << source->count() << endl;
         if (arg("fps") == "auto") outputFps = source->fps();  else outputFps = argd("fps");
         stabilizer->setFrameSource(source);