generic-library/opencv
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Refactored videostab module, added base class for woobble suppression

Browse Source
This commit is contained in:
Alexey Spizhevoy 2012-04-05 09:15:13 +00:00
parent fd9a8c609f
commit f32b645b96
6 changed files with 296 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
modules/videostab/include/opencv2/videostab/global_motion.hpp
View File

@ -105,6 +105,15 @@ protected:
MotionModel motionModel_;
};
class CV_EXPORTS EyeMotionEstimator : public GlobalMotionEstimatorBase
{
public:
virtual Mat estimate(const Mat &/*frame0*/, const Mat &/*frame1*/)
{
return Mat::eye(3, 3, CV_32F);
}
};
class CV_EXPORTS PyrLkRobustMotionEstimator : public GlobalMotionEstimatorBase
{
public:

36
modules/videostab/include/opencv2/videostab/stabilizer.hpp
View File

@ -52,6 +52,7 @@
#include "opencv2/videostab/log.hpp"
#include "opencv2/videostab/inpainting.hpp"
#include "opencv2/videostab/deblurring.hpp"
#include "opencv2/videostab/wobble_suppression.hpp"
namespace cv
{
@ -93,14 +94,14 @@ public:
protected:
StabilizerBase();
void setUp(int cacheSize, const Mat &frame);
void reset();
Mat nextStabilizedFrame();
bool doOneIteration();
void stabilizeFrame(const Mat &stabilizationMotion);
virtual void setUp(Mat &firstFrame) = 0;
virtual void stabilizeFrame() = 0;
virtual void estimateMotion() = 0;
virtual void setUp(const Mat &firstFrame);
virtual Mat estimateMotion() = 0;
virtual Mat estimateStabilizationMotion() = 0;
void stabilizeFrame();
virtual Mat postProcessFrame(const Mat &frame);
Ptr<ILog> log_;
Ptr<IFrameSource> frameSource_;
@ -120,6 +121,7 @@ protected:
Mat preProcessedFrame_;
bool doInpainting_;
Mat inpaintingMask_;
Mat finalFrame_;
std::vector<Mat> frames_;
std::vector<Mat> motions_; // motions_[i] is the motion from i-th to i+1-th frame
std::vector<float> blurrinessRates_;
@ -140,9 +142,10 @@ public:
virtual Mat nextFrame() { return nextStabilizedFrame(); }
private:
virtual void setUp(Mat &firstFrame);
virtual void estimateMotion();
virtual void stabilizeFrame();
virtual void setUp(const Mat &firstFrame);
virtual Mat estimateMotion();
virtual Mat estimateStabilizationMotion();
virtual Mat postProcessFrame(const Mat &frame);
Ptr<MotionFilterBase> motionFilter_;
};
@ -153,7 +156,10 @@ public:
TwoPassStabilizer();
void setMotionStabilizer(Ptr<IMotionStabilizer> val) { motionStabilizer_ = val; }
Ptr<IMotionStabilizer> motionStabilizer() const { return motionStabilizer_; }
Ptr<IMotionStabilizer> motionStabilizer() const { return motionStabilizer_; }
void setWobbleSuppressor(Ptr<WobbleSuppressorBase> val) { wobbleSuppressor_ = val; }
Ptr<WobbleSuppressorBase> wobbleSuppressor() const { return wobbleSuppressor_; }
void setEstimateTrimRatio(bool val) { mustEstTrimRatio_ = val; }
bool mustEstimateTrimaRatio() const { return mustEstTrimRatio_; }
@ -165,18 +171,20 @@ public:
std::vector<Mat> motions() const;
private:
void resetImpl();
void runPrePassIfNecessary();
virtual void setUp(Mat &firstFrame);
virtual void estimateMotion() { /* do nothing as motion was estimation in pre-pass */ }
virtual void stabilizeFrame();
virtual void setUp(const Mat &firstFrame);
virtual Mat estimateMotion();
virtual Mat estimateStabilizationMotion();
virtual Mat postProcessFrame(const Mat &frame);
Ptr<IMotionStabilizer> motionStabilizer_;
Ptr<WobbleSuppressorBase> wobbleSuppressor_;
bool mustEstTrimRatio_;
int frameCount_;
bool isPrePassDone_;
Mat suppressedFrame_;
};
} // namespace videostab

91
modules/videostab/include/opencv2/videostab/wobble_suppression.hpp Normal file
View File

@ -0,0 +1,91 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#ifndef __OPENCV_VIDEOSTAB_WOBBLE_SUPPRESSION_HPP__
#define __OPENCV_VIDEOSTAB_WOBBLE_SUPPRESSION_HPP__
#include <vector>
#include "opencv2/core/core.hpp"
#include "opencv2/videostab/global_motion.hpp"
namespace cv
{
namespace videostab
{
class CV_EXPORTS WobbleSuppressorBase
{
public:
virtual ~WobbleSuppressorBase() {}
virtual void setFrames(const std::vector<Mat> &val) { frames_ = &val; }
virtual const std::vector<Mat>& frames() const { return *frames_; }
virtual void setMotions(const std::vector<Mat> &val) { motions_ = &val; }
virtual const std::vector<Mat>& motions() const { return *motions_; }
virtual void setStabilizedFrames(const std::vector<Mat> &val) { stabilizedFrames_ = &val; }
virtual const std::vector<Mat>& stabilizedFrames() const { return *stabilizedFrames_; }
virtual void setStabilizationMotions(const std::vector<Mat> &val) { stabilizationMotions_ = &val; }
virtual const std::vector<Mat>& stabilizationMotions() const { return *stabilizationMotions_; }
virtual void suppress(int idx, Mat &frame) = 0;
protected:
const std::vector<Mat> *frames_;
const std::vector<Mat> *motions_;
const std::vector<Mat> *stabilizedFrames_;
const std::vector<Mat> *stabilizationMotions_;
};
class CV_EXPORTS NullWobbleSuppressor : public WobbleSuppressorBase
{
public:
virtual void suppress(int idx, Mat &result);
};
} // namespace videostab
} // namespace cv
#endif

183
modules/videostab/src/stabilizer.cpp
View File

@ -65,52 +65,40 @@ StabilizerBase::StabilizerBase()
}
void StabilizerBase::setUp(int cacheSize, const Mat &frame)
void StabilizerBase::reset()
{
InpainterBase *inpainter = static_cast<InpainterBase*>(inpainter_);
doInpainting_ = dynamic_cast<NullInpainter*>(inpainter) == 0;
if (doInpainting_)
{
inpainter_->setMotionModel(motionEstimator_->motionModel());
inpainter_->setFrames(frames_);
inpainter_->setMotions(motions_);
inpainter_->setStabilizedFrames(stabilizedFrames_);
inpainter_->setStabilizationMotions(stabilizationMotions_);
}
DeblurerBase *deblurer = static_cast<DeblurerBase*>(deblurer_);
doDeblurring_ = dynamic_cast<NullDeblurer*>(deblurer) == 0;
if (doDeblurring_)
{
blurrinessRates_.resize(cacheSize);
float blurriness = calcBlurriness(frame);
for (int i = -radius_; i <= 0; ++i)
at(i, blurrinessRates_) = blurriness;
deblurer_->setFrames(frames_);
deblurer_->setMotions(motions_);
deblurer_->setBlurrinessRates(blurrinessRates_);
}
log_->print("processing frames");
frameSize_ = Size(0, 0);
frameMask_ = Mat();
curPos_ = -1;
curStabilizedPos_ = -1;
doDeblurring_ = false;
preProcessedFrame_ = Mat();
doInpainting_ = false;
inpaintingMask_ = Mat();
frames_.clear();
motions_.clear();
blurrinessRates_.clear();
stabilizedFrames_.clear();
stabilizedMasks_.clear();
stabilizationMotions_.clear();
}
Mat StabilizerBase::nextStabilizedFrame()
{
// check if we've processed all frames already
if (curStabilizedPos_ == curPos_ && curStabilizedPos_ != -1)
return Mat(); // we've processed all frames already
return Mat();
bool processed;
do processed = doOneIteration();
while (processed && curStabilizedPos_ == -1);
// check if frame source is empty
if (curStabilizedPos_ == -1)
return Mat(); // frame source is empty
return Mat();
const Mat &stabilizedFrame = at(curStabilizedPos_, stabilizedFrames_);
int dx = static_cast<int>(floor(trimRatio_ * stabilizedFrame.cols));
int dy = static_cast<int>(floor(trimRatio_ * stabilizedFrame.rows));
return stabilizedFrame(Rect(dx, dy, stabilizedFrame.cols - 2*dx, stabilizedFrame.rows - 2*dy));
return postProcessFrame(at(curStabilizedPos_, stabilizedFrames_));
}
@ -128,7 +116,7 @@ bool StabilizerBase::doOneIteration()
if (doDeblurring_)
at(curPos_, blurrinessRates_) = calcBlurriness(frame);
estimateMotion();
at(curPos_ - 1, motions_) = estimateMotion();
if (curPos_ >= radius_)
{
@ -158,15 +146,43 @@ bool StabilizerBase::doOneIteration()
}
void StabilizerBase::stabilizeFrame(const Mat &stabilizationMotion)
void StabilizerBase::setUp(const Mat &firstFrame)
{
Mat stabilizationMotion_;
if (doCorrectionForInclusion_)
stabilizationMotion_ = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_);
else
stabilizationMotion_ = stabilizationMotion.clone();
InpainterBase *inpainter = static_cast<InpainterBase*>(inpainter_);
doInpainting_ = dynamic_cast<NullInpainter*>(inpainter) == 0;
if (doInpainting_)
{
inpainter_->setMotionModel(motionEstimator_->motionModel());
inpainter_->setFrames(frames_);
inpainter_->setMotions(motions_);
inpainter_->setStabilizedFrames(stabilizedFrames_);
inpainter_->setStabilizationMotions(stabilizationMotions_);
}
at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion_;
DeblurerBase *deblurer = static_cast<DeblurerBase*>(deblurer_);
doDeblurring_ = dynamic_cast<NullDeblurer*>(deblurer) == 0;
if (doDeblurring_)
{
blurrinessRates_.resize(2*radius_ + 1);
float blurriness = calcBlurriness(firstFrame);
for (int i = -radius_; i <= 0; ++i)
at(i, blurrinessRates_) = blurriness;
deblurer_->setFrames(frames_);
deblurer_->setMotions(motions_);
deblurer_->setBlurrinessRates(blurrinessRates_);
}
log_->print("processing frames");
}
void StabilizerBase::stabilizeFrame()
{
Mat stabilizationMotion = estimateStabilizationMotion();
if (doCorrectionForInclusion_)
stabilizationMotion = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_);
at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion;
if (doDeblurring_)
{
@ -181,22 +197,22 @@ void StabilizerBase::stabilizeFrame(const Mat &stabilizationMotion)
if (motionEstimator_->motionModel() != HOMOGRAPHY)
warpAffine(
preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_),
stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_LINEAR, borderMode_);
stabilizationMotion(Rect(0,0,3,2)), frameSize_, INTER_LINEAR, borderMode_);
else
warpPerspective(
preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_),
stabilizationMotion_, frameSize_, INTER_LINEAR, borderMode_);
stabilizationMotion, frameSize_, INTER_LINEAR, borderMode_);
if (doInpainting_)
{
if (motionEstimator_->motionModel() != HOMOGRAPHY)
warpAffine(
frameMask_, at(curStabilizedPos_, stabilizedMasks_),
stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_NEAREST);
stabilizationMotion(Rect(0,0,3,2)), frameSize_, INTER_NEAREST);
else
warpPerspective(
frameMask_, at(curStabilizedPos_, stabilizedMasks_),
stabilizationMotion_, frameSize_, INTER_NEAREST);
stabilizationMotion, frameSize_, INTER_NEAREST);
erode(at(curStabilizedPos_, stabilizedMasks_), at(curStabilizedPos_, stabilizedMasks_),
Mat());
@ -209,6 +225,15 @@ void StabilizerBase::stabilizeFrame(const Mat &stabilizationMotion)
}
Mat StabilizerBase::postProcessFrame(const Mat &frame)
{
// trim frame
int dx = static_cast<int>(floor(trimRatio_ * frame.cols));
int dy = static_cast<int>(floor(trimRatio_ * frame.rows));
return frame(Rect(dx, dy, frame.cols - 2*dx, frame.rows - 2*dy));
}
OnePassStabilizer::OnePassStabilizer()
{
setMotionFilter(new GaussianMotionFilter());
@ -218,25 +243,17 @@ OnePassStabilizer::OnePassStabilizer()
void OnePassStabilizer::reset()
{
curPos_ = -1;
curStabilizedPos_ = -1;
frames_.clear();
motions_.clear();
stabilizedFrames_.clear();
stabilizationMotions_.clear();
doDeblurring_ = false;
doInpainting_ = false;
StabilizerBase::reset();
}
void OnePassStabilizer::setUp(Mat &firstFrame)
void OnePassStabilizer::setUp(const Mat &firstFrame)
{
frameSize_ = firstFrame.size();
frameMask_.create(frameSize_, CV_8U);
frameMask_.setTo(255);
int cacheSize = 2*radius_ + 1;
frames_.resize(cacheSize);
stabilizedFrames_.resize(cacheSize);
stabilizedMasks_.resize(cacheSize);
@ -251,27 +268,32 @@ void OnePassStabilizer::setUp(Mat &firstFrame)
at(0, frames_) = firstFrame;
StabilizerBase::setUp(cacheSize, firstFrame);
StabilizerBase::setUp(firstFrame);
}
void OnePassStabilizer::estimateMotion()
Mat OnePassStabilizer::estimateMotion()
{
at(curPos_ - 1, motions_) = motionEstimator_->estimate(
at(curPos_ - 1, frames_), at(curPos_, frames_));
return motionEstimator_->estimate(at(curPos_ - 1, frames_), at(curPos_, frames_));
}
void OnePassStabilizer::stabilizeFrame()
Mat OnePassStabilizer::estimateStabilizationMotion()
{
Mat stabilizationMotion = motionFilter_->stabilize(curStabilizedPos_, motions_, make_pair(0, curPos_));
StabilizerBase::stabilizeFrame(stabilizationMotion);
return motionFilter_->stabilize(curStabilizedPos_, motions_, make_pair(0, curPos_));
}
Mat OnePassStabilizer::postProcessFrame(const Mat &frame)
{
return StabilizerBase::postProcessFrame(frame);
}
TwoPassStabilizer::TwoPassStabilizer()
{
setMotionStabilizer(new GaussianMotionFilter());
setWobbleSuppressor(new NullWobbleSuppressor());
setEstimateTrimRatio(false);
reset();
}
@ -279,16 +301,10 @@ TwoPassStabilizer::TwoPassStabilizer()
void TwoPassStabilizer::reset()
{
isPrePassDone_ = false;
StabilizerBase::reset();
frameCount_ = 0;
curPos_ = -1;
curStabilizedPos_ = -1;
frames_.clear();
motions_.clear();
stabilizedFrames_.clear();
stabilizationMotions_.clear();
doDeblurring_ = false;
doInpainting_ = false;
isPrePassDone_ = false;
suppressedFrame_ = Mat();
}
@ -360,10 +376,9 @@ void TwoPassStabilizer::runPrePassIfNecessary()
}
void TwoPassStabilizer::setUp(Mat &firstFrame)
void TwoPassStabilizer::setUp(const Mat &firstFrame)
{
int cacheSize = 2*radius_ + 1;
frames_.resize(cacheSize);
stabilizedFrames_.resize(cacheSize);
stabilizedMasks_.resize(cacheSize);
@ -371,13 +386,31 @@ void TwoPassStabilizer::setUp(Mat &firstFrame)
for (int i = -radius_; i <= 0; ++i)
at(i, frames_) = firstFrame;
StabilizerBase::setUp(cacheSize, firstFrame);
wobbleSuppressor_->setFrames(frames_);
wobbleSuppressor_->setMotions(motions_);
wobbleSuppressor_->setStabilizedFrames(stabilizedFrames_);
wobbleSuppressor_->setStabilizationMotions(stabilizationMotions_);
StabilizerBase::setUp(firstFrame);
}
void TwoPassStabilizer::stabilizeFrame()
Mat TwoPassStabilizer::estimateMotion()
{
StabilizerBase::stabilizeFrame(stabilizationMotions_[curStabilizedPos_]);
return motions_[curPos_ - 1].clone();
}
Mat TwoPassStabilizer::estimateStabilizationMotion()
{
return stabilizationMotions_[curStabilizedPos_].clone();
}
Mat TwoPassStabilizer::postProcessFrame(const Mat &/*frame*/)
{
wobbleSuppressor_->suppress(curStabilizedPos_, suppressedFrame_);
return StabilizerBase::postProcessFrame(suppressedFrame_);
}
} // namespace videostab

60
modules/videostab/src/wobble_suppression.cpp Normal file
View File

@ -0,0 +1,60 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#include "precomp.hpp"
#include "opencv2/videostab/wobble_suppression.hpp"
#include "opencv2/videostab/ring_buffer.hpp"
using namespace std;
namespace cv
{
namespace videostab
{
void NullWobbleSuppressor::suppress(int idx, Mat &result)
{
result = at(idx, *stabilizedFrames_);
}
} // namespace videostab
} // namespace cv

8
samples/cpp/videostab.cpp
View File

@ -71,9 +71,11 @@ void run()
{
VideoWriter writer;
Mat stabilizedFrame;
int nframes = 0;
while (!(stabilizedFrame = stabilizedFrames->nextFrame()).empty())
{
nframes++;
if (!saveMotionsPath.empty())
saveMotionsIfNecessary();
if (!outputPath.empty())
@ -91,7 +93,9 @@ void run()
}
}
cout << "\nfinished\n";
cout << endl
<< "processed frames: " << nframes << endl
<< "finished\n";
}
@ -248,7 +252,7 @@ int main(int argc, const char **argv)
if (inputPath.empty()) throw runtime_error("specify video file path");
VideoFileSource *source = new VideoFileSource(inputPath);
cout << "frame count: " << source->count() << endl;
cout << "frame count (rough): " << source->count() << endl;
if (arg("fps") == "auto") outputFps = source->fps(); else outputFps = argd("fps");
stabilizer->setFrameSource(source);