From dcb5464b3c5a4a090515bd78bcc41a1e8af61a94 Mon Sep 17 00:00:00 2001
From: Alexey Spizhevoy <no@email>
Date: Tue, 20 Mar 2012 12:24:51 +0000
Subject: [PATCH] Added more inpainting methods. Fixed some errors.
---
modules/videostab/CMakeLists.txt | 2 +-
.../opencv2/videostab/frame_source.hpp | 1 +
.../include/opencv2/videostab/inpainting.hpp | 15 +++++
modules/videostab/src/inpainting.cpp | 66 +++++++++++++++----
samples/cpp/videostab.cpp | 41 ++++++++++--
5 files changed, 104 insertions(+), 21 deletions(-)
diff --git a/modules/videostab/CMakeLists.txt b/modules/videostab/CMakeLists.txt
index 56dbae2c7..ed7f8c45c 100644
--- a/modules/videostab/CMakeLists.txt
+++ b/modules/videostab/CMakeLists.txt
@@ -1,3 +1,3 @@
set(the_description "Video stabilization")
-ocv_define_module(videostab opencv_imgproc opencv_features2d opencv_video opencv_highgui OPTIONAL opencv_gpu)
+ocv_define_module(videostab opencv_imgproc opencv_features2d opencv_video opencv_highgui opencv_photo OPTIONAL opencv_gpu)
diff --git a/modules/videostab/include/opencv2/videostab/frame_source.hpp b/modules/videostab/include/opencv2/videostab/frame_source.hpp
index ff83a681e..24facb5cf 100644
--- a/modules/videostab/include/opencv2/videostab/frame_source.hpp
+++ b/modules/videostab/include/opencv2/videostab/frame_source.hpp
@@ -72,6 +72,7 @@ class CV_EXPORTS VideoFileSource : public IFrameSource
{
public:
VideoFileSource(const std::string &path, bool volatileFrame = false);
+
virtual void reset();
virtual Mat nextFrame();
diff --git a/modules/videostab/include/opencv2/videostab/inpainting.hpp b/modules/videostab/include/opencv2/videostab/inpainting.hpp
index fd65eefaa..1d83978ab 100644
--- a/modules/videostab/include/opencv2/videostab/inpainting.hpp
+++ b/modules/videostab/include/opencv2/videostab/inpainting.hpp
@@ -47,6 +47,7 @@
#include "opencv2/core/core.hpp"
#include "opencv2/videostab/optical_flow.hpp"
#include "opencv2/videostab/fast_marching.hpp"
+#include "opencv2/photo/photo.hpp"
namespace cv
{
@@ -163,6 +164,20 @@ private:
FastMarchingMethod fmm_;
};
+class CV_EXPORTS ColorInpainter : public IInpainter
+{
+public:
+ ColorInpainter(int method = INPAINT_TELEA, double radius = 2.)
+ : method_(method), radius_(radius) {}
+
+ virtual void inpaint(int idx, Mat &frame, Mat &mask);
+
+private:
+ int method_;
+ double radius_;
+ Mat invMask_;
+};
+
CV_EXPORTS void calcFlowMask(
const Mat &flowX, const Mat &flowY, const Mat &errors, float maxError,
const Mat &mask0, const Mat &mask1, Mat &flowMask);
diff --git a/modules/videostab/src/inpainting.cpp b/modules/videostab/src/inpainting.cpp
index db12cb345..c7452dd4d 100644
--- a/modules/videostab/src/inpainting.cpp
+++ b/modules/videostab/src/inpainting.cpp
@@ -110,7 +110,7 @@ struct Pixel3
ConsistentMosaicInpainter::ConsistentMosaicInpainter()
{
- setStdevThresh(10);
+ setStdevThresh(20);
}
@@ -190,24 +190,17 @@ class MotionInpaintBody
public:
void operator ()(int x, int y)
{
- float uEst = 0.f, vEst = 0.f;
- float wSum = 0.f;
+ float uEst = 0.f, vEst = 0.f, wSum = 0.f;
for (int dy = -rad; dy <= rad; ++dy)
{
for (int dx = -rad; dx <= rad; ++dx)
{
int qx0 = x + dx;
- int qy0 = y + dy;
+ int qy0 = y + dy;
- if (qy0 > 0 && qy0+1 < mask0.rows && qx0 > 0 && qx0+1 < mask0.cols && mask0(qy0,qx0) &&
- mask0(qy0-1,qx0) && mask0(qy0+1,qx0) && mask0(qy0,qx0-1) && mask0(qy0,qx0+1))
+ if (qy0 >= 0 && qy0 < mask0.rows && qx0 >= 0 && qx0 < mask0.cols && mask0(qy0,qx0))
{
- float dudx = 0.5f * (flowX(qy0,qx0+1) - flowX(qy0,qx0-1));
- float dvdx = 0.5f * (flowY(qy0,qx0+1) - flowY(qy0,qx0-1));
- float dudy = 0.5f * (flowX(qy0+1,qx0) - flowX(qy0-1,qx0));
- float dvdy = 0.5f * (flowY(qy0+1,qx0) - flowY(qy0-1,qx0));
-
int qx1 = cvRound(qx0 + flowX(qy0,qx0));
int qy1 = cvRound(qy0 + flowY(qy0,qx0));
int px1 = qx1 - dx;
@@ -216,14 +209,54 @@ public:
if (qx1 >= 0 && qx1 < mask1.cols && qy1 >= 0 && qy1 < mask1.rows && mask1(qy1,qx1) &&
px1 >= 0 && px1 < mask1.cols && py1 >= 0 && py1 < mask1.rows && mask1(py1,px1))
{
+ float dudx = 0.f, dvdx = 0.f, dudy = 0.f, dvdy = 0.f;
+
+ if (qx0 > 0 && mask0(qy0,qx0-1))
+ {
+ if (qx0+1 < mask0.cols && mask0(qy0,qx0+1))
+ {
+ dudx = (flowX(qy0,qx0+1) - flowX(qy0,qx0-1)) * 0.5f;
+ dvdx = (flowY(qy0,qx0+1) - flowY(qy0,qx0-1)) * 0.5f;
+ }
+ else
+ {
+ dudx = flowX(qy0,qx0) - flowX(qy0,qx0-1);
+ dvdx = flowY(qy0,qx0) - flowY(qy0,qx0-1);
+ }
+ }
+ else if (qx0+1 < mask0.cols && mask0(qy0,qx0+1))
+ {
+ dudx = flowX(qy0,qx0+1) - flowX(qy0,qx0);
+ dvdx = flowY(qy0,qx0+1) - flowY(qy0,qx0);
+ }
+
+ if (qy0 > 0 && mask0(qy0-1,qx0))
+ {
+ if (qy0+1 < mask0.rows && mask0(qy0+1,qx0))
+ {
+ dudy = (flowX(qy0+1,qx0) - flowX(qy0-1,qx0)) * 0.5f;
+ dvdy = (flowY(qy0+1,qx0) - flowY(qy0-1,qx0)) * 0.5f;
+ }
+ else
+ {
+ dudy = flowX(qy0,qx0) - flowX(qy0-1,qx0);
+ dvdy = flowY(qy0,qx0) - flowY(qy0-1,qx0);
+ }
+ }
+ else if (qy0+1 < mask0.rows && mask0(qy0+1,qx0))
+ {
+ dudy = flowX(qy0+1,qx0) - flowX(qy0,qx0);
+ dvdy = flowY(qy0+1,qx0) - flowY(qy0,qx0);
+ }
+
Point3_<uchar> cp = frame1(py1,px1), cq = frame1(qy1,qx1);
float distColor = sqr(cp.x-cq.x) + sqr(cp.y-cq.y) + sqr(cp.z-cq.z);
float w = 1.f / (sqrt(distColor * (dx*dx + dy*dy)) + eps);
+
uEst += w * (flowX(qy0,qx0) - dudx*dx - dudy*dy);
vEst += w * (flowY(qy0,qx0) - dvdx*dx - dvdy*dy);
wSum += w;
}
- //else return;
}
}
}
@@ -317,7 +350,7 @@ void MotionInpainter::inpaint(int idx, Mat &frame, Mat &mask)
fmm_.run(flowMask_, body);
completeFrameAccordingToFlow(
- flowMask_, flowX_, flowY_, transformedFrame1_, transformedMask1_, frame, mask);
+ flowMask_, flowX_, flowY_, transformedFrame1_, transformedMask1_, frame, mask);
}
}
@@ -364,6 +397,13 @@ void ColorAverageInpainter::inpaint(int /*idx*/, Mat &frame, Mat &mask)
}
+void ColorInpainter::inpaint(int /*idx*/, Mat &frame, Mat &mask)
+{
+ bitwise_not(mask, invMask_);
+ cv::inpaint(frame, invMask_, frame, radius_, method_);
+}
+
+
void calcFlowMask(
const Mat &flowX, const Mat &flowY, const Mat &errors, float maxError,
const Mat &mask0, const Mat &mask1, Mat &flowMask)
diff --git a/samples/cpp/videostab.cpp b/samples/cpp/videostab.cpp
index 6616624a8..68b62192d 100644
--- a/samples/cpp/videostab.cpp
+++ b/samples/cpp/videostab.cpp
@@ -73,7 +73,7 @@ void printHelp()
" you can turn it off if you want to use one pass only).\n"
" --incl-constr=(yes|no)\n"
" Ensure the inclusion constraint is always satisfied. The default is no.\n"
- " --border-mode=(replicate|const)\n"
+ " --border-mode=(replicate|reflect|const)\n"
" Set border extrapolation mode. The default is replicate.\n"
" --mosaic=(yes|no)\n"
" Do consistent mosaicing. The default is no.\n"
@@ -81,8 +81,10 @@ void printHelp()
" Consistent mosaicing stdev threshold. The default is 10.\n"
" --motion-inpaint=(yes|no)\n"
" Do motion inpainting (requires GPU support). The default is no.\n"
- " --color-inpaint=(yes|no)\n"
+ " --color-inpaint=(no|average|ns|telea)\n"
" Do color inpainting. The defailt is no.\n"
+ " --color-inpaint-radius=<float_number>\n"
+ " Set color inpainting radius (for ns and telea options only).\n"
" -o, --output=<file_path>\n"
" Set output file path explicitely. The default is stabilized.avi.\n"
" --fps=<int_number>\n"
@@ -115,7 +117,8 @@ int main(int argc, const char **argv)
"{ | mosaic | | }"
"{ | mosaic-stdev | | }"
"{ | motion-inpaint | | }"
- "{ | color-inpaint | | }"
+ "{ | color-inpaint | no | }"
+ "{ | color-inpaint-radius | | }"
"{ o | output | stabilized.avi | }"
"{ | fps | | }"
"{ q | quiet | false | }"
@@ -172,7 +175,7 @@ int main(int argc, const char **argv)
if (smoothRadius > 0 && smoothStdev > 0)
stabilizer->setMotionFilter(new GaussianMotionFilter(smoothRadius, smoothStdev));
else if (smoothRadius > 0 && smoothStdev < 0)
- stabilizer->setMotionFilter(new GaussianMotionFilter(smoothRadius, sqrt(smoothRadius)));
+ stabilizer->setMotionFilter(new GaussianMotionFilter(smoothRadius, sqrt(static_cast<float>(smoothRadius))));
if (cmd.get<string>("deblur") == "yes")
{
@@ -191,7 +194,9 @@ int main(int argc, const char **argv)
if (!cmd.get<string>("incl-constr").empty())
stabilizer->setInclusionConstraint(cmd.get<string>("incl-constr") == "yes");
- if (cmd.get<string>("border-mode") == "replicate")
+ if (cmd.get<string>("border-mode") == "reflect")
+ stabilizer->setBorderMode(BORDER_REFLECT);
+ else if (cmd.get<string>("border-mode") == "replicate")
stabilizer->setBorderMode(BORDER_REPLICATE);
else if (cmd.get<string>("border-mode") == "const")
stabilizer->setBorderMode(BORDER_CONSTANT);
@@ -208,8 +213,30 @@ int main(int argc, const char **argv)
}
if (cmd.get<string>("motion-inpaint") == "yes")
inpainters->pushBack(new MotionInpainter());
- if (cmd.get<string>("color-inpaint") == "yes")
- inpainters->pushBack(new ColorAverageInpainter());
+ if (!cmd.get<string>("color-inpaint").empty())
+ {
+ if (cmd.get<string>("color-inpaint") == "average")
+ inpainters->pushBack(new ColorAverageInpainter());
+ else if (!cmd.get<string>("color-inpaint-radius").empty())
+ {
+ float radius = cmd.get<float>("color-inpaint-radius");
+ if (cmd.get<string>("color-inpaint") == "ns")
+ inpainters->pushBack(new ColorInpainter(INPAINT_NS, radius));
+ else if (cmd.get<string>("color-inpaint") == "telea")
+ inpainters->pushBack(new ColorInpainter(INPAINT_TELEA, radius));
+ else if (cmd.get<string>("color-inpaint") != "no")
+ throw runtime_error("unknown color inpainting method: " + cmd.get<string>("color-inpaint"));
+ }
+ else
+ {
+ if (cmd.get<string>("color-inpaint") == "ns")
+ inpainters->pushBack(new ColorInpainter(INPAINT_NS));
+ else if (cmd.get<string>("color-inpaint") == "telea")
+ inpainters->pushBack(new ColorInpainter(INPAINT_TELEA));
+ else if (cmd.get<string>("color-inpaint") != "no")
+ throw runtime_error("unknown color inpainting method: " + cmd.get<string>("color-inpaint"));
+ }
+ }
if (!inpainters->empty())
stabilizer->setInpainter(inpainters);