From 37ea8722048187ea409541e8f2c9517b52fb1321 Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Wed, 3 Jul 2013 15:20:14 +0400 Subject: [PATCH 1/7] added handling of camera parameters estimation errors (#3122) --- .../stitching/include/opencv2/stitching.hpp | 10 +++++-- .../stitching/detail/motion_estimators.hpp | 26 ++++++++++++------- modules/stitching/src/motion_estimators.cpp | 26 ++++++++++++++++--- modules/stitching/src/stitcher.cpp | 13 +++++++--- samples/cpp/stitching_detailed.cpp | 12 +++++++-- 5 files changed, 65 insertions(+), 22 deletions(-) diff --git a/modules/stitching/include/opencv2/stitching.hpp b/modules/stitching/include/opencv2/stitching.hpp index 3ccd2878a..2c48f2f2a 100644 --- a/modules/stitching/include/opencv2/stitching.hpp +++ b/modules/stitching/include/opencv2/stitching.hpp @@ -59,7 +59,13 @@ class CV_EXPORTS Stitcher { public: enum { ORIG_RESOL = -1 }; - enum Status { OK, ERR_NEED_MORE_IMGS }; + enum Status + { + OK = 0, + ERR_NEED_MORE_IMGS = 1, + ERR_HOMOGRAPHY_EST_FAIL = 2, + ERR_CAMERA_PARAMS_ADJUST_FAIL = 3 + }; // Creates stitcher with default parameters static Stitcher createDefault(bool try_use_gpu = false); @@ -138,7 +144,7 @@ private: Stitcher() {} Status matchImages(); - void estimateCameraParams(); + Status estimateCameraParams(); double registr_resol_; double seam_est_resol_; diff --git a/modules/stitching/include/opencv2/stitching/detail/motion_estimators.hpp b/modules/stitching/include/opencv2/stitching/detail/motion_estimators.hpp index 581a0292d..c0e446c60 100644 --- a/modules/stitching/include/opencv2/stitching/detail/motion_estimators.hpp +++ b/modules/stitching/include/opencv2/stitching/detail/motion_estimators.hpp @@ -56,12 +56,14 @@ class CV_EXPORTS Estimator public: virtual ~Estimator() {} - void operator ()(const std::vector &features, const std::vector &pairwise_matches, + bool operator ()(const std::vector &features, + const std::vector &pairwise_matches, std::vector &cameras) - { estimate(features, pairwise_matches, cameras); } + { return estimate(features, pairwise_matches, cameras); } protected: - virtual void estimate(const std::vector &features, const std::vector &pairwise_matches, + virtual bool estimate(const std::vector &features, + const std::vector &pairwise_matches, std::vector &cameras) = 0; }; @@ -73,8 +75,9 @@ public: : is_focals_estimated_(is_focals_estimated) {} private: - void estimate(const std::vector &features, const std::vector &pairwise_matches, - std::vector &cameras); + virtual bool estimate(const std::vector &features, + const std::vector &pairwise_matches, + std::vector &cameras); bool is_focals_estimated_; }; @@ -107,7 +110,7 @@ protected: } // Runs bundle adjustment - virtual void estimate(const std::vector &features, + virtual bool estimate(const std::vector &features, const std::vector &pairwise_matches, std::vector &cameras); @@ -193,11 +196,14 @@ void CV_EXPORTS waveCorrect(std::vector &rmats, WaveCorrectKind kind); String CV_EXPORTS matchesGraphAsString(std::vector &pathes, std::vector &pairwise_matches, float conf_threshold); -std::vector CV_EXPORTS leaveBiggestComponent(std::vector &features, std::vector &pairwise_matches, - float conf_threshold); +std::vector CV_EXPORTS leaveBiggestComponent( + std::vector &features, + std::vector &pairwise_matches, + float conf_threshold); -void CV_EXPORTS findMaxSpanningTree(int num_images, const std::vector &pairwise_matches, - Graph &span_tree, std::vector ¢ers); +void CV_EXPORTS findMaxSpanningTree( + int num_images, const std::vector &pairwise_matches, + Graph &span_tree, std::vector ¢ers); } // namespace detail } // namespace cv diff --git a/modules/stitching/src/motion_estimators.cpp b/modules/stitching/src/motion_estimators.cpp index 1bb3df7bb..3640bd3cc 100644 --- a/modules/stitching/src/motion_estimators.cpp +++ b/modules/stitching/src/motion_estimators.cpp @@ -101,8 +101,10 @@ void calcDeriv(const Mat &err1, const Mat &err2, double h, Mat res) namespace cv { namespace detail { -void HomographyBasedEstimator::estimate(const std::vector &features, const std::vector &pairwise_matches, - std::vector &cameras) +bool HomographyBasedEstimator::estimate( + const std::vector &features, + const std::vector &pairwise_matches, + std::vector &cameras) { LOGLN("Estimating rotations..."); #if ENABLE_LOG @@ -164,12 +166,13 @@ void HomographyBasedEstimator::estimate(const std::vector &featur } LOGLN("Estimating rotations, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); + return true; } ////////////////////////////////////////////////////////////////////////////// -void BundleAdjusterBase::estimate(const std::vector &features, +bool BundleAdjusterBase::estimate(const std::vector &features, const std::vector &pairwise_matches, std::vector &cameras) { @@ -245,7 +248,21 @@ void BundleAdjusterBase::estimate(const std::vector &features, LOGLN_CHAT("Bundle adjustment, final RMS error: " << std::sqrt(err.dot(err) / total_num_matches_)); LOGLN_CHAT("Bundle adjustment, iterations done: " << iter); - obtainRefinedCameraParams(cameras); + // Check if all camera parameters are valid + bool ok = true; + for (int i = 0; i < cam_params_.rows; ++i) + { + if (isnan(cam_params_.at(i,0)) || + isinf(cam_params_.at(i,0))) + { + ok = false; + break; + } + } + if (!ok) + return false; + + obtainRefinedCameraParams(cameras); // Normalize motion to center image Graph span_tree; @@ -256,6 +273,7 @@ void BundleAdjusterBase::estimate(const std::vector &features, cameras[i].R = R_inv * cameras[i].R; LOGLN_CHAT("Bundle adjustment, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); + return true; } diff --git a/modules/stitching/src/stitcher.cpp b/modules/stitching/src/stitcher.cpp index cef9fe4e9..75307209d 100644 --- a/modules/stitching/src/stitcher.cpp +++ b/modules/stitching/src/stitcher.cpp @@ -102,7 +102,8 @@ Stitcher::Status Stitcher::estimateTransform(InputArray images, const std::vecto if ((status = matchImages()) != OK) return status; - estimateCameraParams(); + if ((status = estimateCameraParams()) != OK) + return status; return OK; } @@ -442,10 +443,11 @@ Stitcher::Status Stitcher::matchImages() } -void Stitcher::estimateCameraParams() +Stitcher::Status Stitcher::estimateCameraParams() { detail::HomographyBasedEstimator estimator; - estimator(features_, pairwise_matches_, cameras_); + if (!estimator(features_, pairwise_matches_, cameras_)) + return ERR_HOMOGRAPHY_EST_FAIL; for (size_t i = 0; i < cameras_.size(); ++i) { @@ -456,7 +458,8 @@ void Stitcher::estimateCameraParams() } bundle_adjuster_->setConfThresh(conf_thresh_); - (*bundle_adjuster_)(features_, pairwise_matches_, cameras_); + if (!(*bundle_adjuster_)(features_, pairwise_matches_, cameras_)) + return ERR_CAMERA_PARAMS_ADJUST_FAIL; // Find median focal length and use it as final image scale std::vector focals; @@ -481,6 +484,8 @@ void Stitcher::estimateCameraParams() for (size_t i = 0; i < cameras_.size(); ++i) cameras_[i].R = rmats[i]; } + + return OK; } } // namespace cv diff --git a/samples/cpp/stitching_detailed.cpp b/samples/cpp/stitching_detailed.cpp index 5ee79184d..a50229a57 100644 --- a/samples/cpp/stitching_detailed.cpp +++ b/samples/cpp/stitching_detailed.cpp @@ -469,7 +469,11 @@ int main(int argc, char* argv[]) HomographyBasedEstimator estimator; vector cameras; - estimator(features, pairwise_matches, cameras); + if (!estimator(features, pairwise_matches, cameras)) + { + cout << "Homography estimation failed.\n"; + return -1; + } for (size_t i = 0; i < cameras.size(); ++i) { @@ -495,7 +499,11 @@ int main(int argc, char* argv[]) if (ba_refine_mask[3] == 'x') refine_mask(1,1) = 1; if (ba_refine_mask[4] == 'x') refine_mask(1,2) = 1; adjuster->setRefinementMask(refine_mask); - (*adjuster)(features, pairwise_matches, cameras); + if (!(*adjuster)(features, pairwise_matches, cameras)) + { + cout << "Camera parameters adjusting failed.\n"; + return -1; + } // Find median focal length From 438a599288e003b516b9d41838b3d280a1dc6b6e Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Thu, 4 Jul 2013 16:57:20 +0400 Subject: [PATCH 2/7] added 'cmath' include --- modules/stitching/src/motion_estimators.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/stitching/src/motion_estimators.cpp b/modules/stitching/src/motion_estimators.cpp index 3640bd3cc..494104d23 100644 --- a/modules/stitching/src/motion_estimators.cpp +++ b/modules/stitching/src/motion_estimators.cpp @@ -41,6 +41,7 @@ //M*/ #include "precomp.hpp" +#include #include "opencv2/calib3d/calib3d_c.h" using namespace cv; @@ -252,8 +253,8 @@ bool BundleAdjusterBase::estimate(const std::vector &features, bool ok = true; for (int i = 0; i < cam_params_.rows; ++i) { - if (isnan(cam_params_.at(i,0)) || - isinf(cam_params_.at(i,0))) + if (std::isnan(cam_params_.at(i,0)) || + std::isinf(cam_params_.at(i,0))) { ok = false; break; From cc85e86ee7a65ed26ed49a1998130b6da829dd24 Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Thu, 4 Jul 2013 17:22:36 +0400 Subject: [PATCH 3/7] fixed Windows build --- modules/stitching/src/motion_estimators.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/stitching/src/motion_estimators.cpp b/modules/stitching/src/motion_estimators.cpp index 494104d23..fd705fb02 100644 --- a/modules/stitching/src/motion_estimators.cpp +++ b/modules/stitching/src/motion_estimators.cpp @@ -41,7 +41,7 @@ //M*/ #include "precomp.hpp" -#include +#include #include "opencv2/calib3d/calib3d_c.h" using namespace cv; @@ -253,8 +253,8 @@ bool BundleAdjusterBase::estimate(const std::vector &features, bool ok = true; for (int i = 0; i < cam_params_.rows; ++i) { - if (std::isnan(cam_params_.at(i,0)) || - std::isinf(cam_params_.at(i,0))) + if (isnan(cam_params_.at(i,0)) || + isinf(cam_params_.at(i,0))) { ok = false; break; From 027065a5a131e7d41aa15e860997c32c46bee64f Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Thu, 4 Jul 2013 17:31:12 +0400 Subject: [PATCH 4/7] removed trailing whitespaces causing warnings --- modules/stitching/src/motion_estimators.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/stitching/src/motion_estimators.cpp b/modules/stitching/src/motion_estimators.cpp index fd705fb02..026bea6fc 100644 --- a/modules/stitching/src/motion_estimators.cpp +++ b/modules/stitching/src/motion_estimators.cpp @@ -263,7 +263,7 @@ bool BundleAdjusterBase::estimate(const std::vector &features, if (!ok) return false; - obtainRefinedCameraParams(cameras); + obtainRefinedCameraParams(cameras); // Normalize motion to center image Graph span_tree; From 406f342dc339a8c44ad0de628ca5035994497eaa Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Thu, 4 Jul 2013 17:41:39 +0400 Subject: [PATCH 5/7] updated docs --- modules/stitching/doc/motion_estimation.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/modules/stitching/doc/motion_estimation.rst b/modules/stitching/doc/motion_estimation.rst index 6cda27522..d8ef015f7 100644 --- a/modules/stitching/doc/motion_estimation.rst +++ b/modules/stitching/doc/motion_estimation.rst @@ -18,12 +18,11 @@ Rotation estimator base class. It takes features of all images, pairwise matches public: virtual ~Estimator() {} - void operator ()(const std::vector &features, const std::vector &pairwise_matches, - std::vector &cameras) - { estimate(features, pairwise_matches, cameras); } + bool operator ()(const std::vector &features, const std::vector &pairwise_matches, + std::vector &cameras); protected: - virtual void estimate(const std::vector &features, const std::vector &pairwise_matches, + virtual bool estimate(const std::vector &features, const std::vector &pairwise_matches, std::vector &cameras) = 0; }; @@ -40,6 +39,8 @@ Estimates camera parameters. :param cameras: Estimated camera parameters + :return: True in case of success, false otherwise + detail::Estimator::estimate --------------------------- @@ -53,6 +54,8 @@ This method must implement camera parameters estimation logic in order to make t :param cameras: Estimated camera parameters + :return: True in case of success, false otherwise + detail::HomographyBasedEstimator -------------------------------- .. ocv:class:: detail::HomographyBasedEstimator : public detail::Estimator From 2a48f7c0350bfb2a3d2d05a1b45200d4f870c863 Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Fri, 5 Jul 2013 09:44:32 +0400 Subject: [PATCH 6/7] fixed include for Windows --- modules/stitching/src/motion_estimators.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/stitching/src/motion_estimators.cpp b/modules/stitching/src/motion_estimators.cpp index 026bea6fc..abd43b11f 100644 --- a/modules/stitching/src/motion_estimators.cpp +++ b/modules/stitching/src/motion_estimators.cpp @@ -41,9 +41,15 @@ //M*/ #include "precomp.hpp" -#include #include "opencv2/calib3d/calib3d_c.h" +#ifdef _MSC_VER + #include + #define isnan(x) _isnan(x) +#else + #include +#endif + using namespace cv; using namespace cv::detail; @@ -253,8 +259,7 @@ bool BundleAdjusterBase::estimate(const std::vector &features, bool ok = true; for (int i = 0; i < cam_params_.rows; ++i) { - if (isnan(cam_params_.at(i,0)) || - isinf(cam_params_.at(i,0))) + if (isnan(cam_params_.at(i,0))) { ok = false; break; From dcb049df6adbafe1a3184e37e205994e994f189a Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Fri, 5 Jul 2013 10:16:22 +0400 Subject: [PATCH 7/7] fixed docs --- modules/stitching/doc/motion_estimation.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/stitching/doc/motion_estimation.rst b/modules/stitching/doc/motion_estimation.rst index d8ef015f7..5859f3e02 100644 --- a/modules/stitching/doc/motion_estimation.rst +++ b/modules/stitching/doc/motion_estimation.rst @@ -31,7 +31,7 @@ detail::Estimator::operator() Estimates camera parameters. -.. ocv:function:: detail::Estimator::operator ()(const std::vector &features, const std::vector &pairwise_matches, std::vector &cameras) +.. ocv:function:: bool detail::Estimator::operator ()(const std::vector &features, const std::vector &pairwise_matches, std::vector &cameras) :param features: Features of images @@ -46,7 +46,7 @@ detail::Estimator::estimate This method must implement camera parameters estimation logic in order to make the wrapper `detail::Estimator::operator()`_ work. -.. ocv:function:: void detail::Estimator::estimate(const std::vector &features, const std::vector &pairwise_matches, std::vector &cameras) +.. ocv:function:: bool detail::Estimator::estimate(const std::vector &features, const std::vector &pairwise_matches, std::vector &cameras) :param features: Features of images