From 551113292bba1f76016227cb05d1083c0b24be71 Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Mon, 26 Sep 2011 08:52:31 +0000 Subject: [PATCH] Refactored stitching module --- .../include/opencv2/stitching/stitcher.hpp | 29 ++- modules/stitching/src/stitcher.cpp | 233 ++++++++++-------- 2 files changed, 153 insertions(+), 109 deletions(-) diff --git a/modules/stitching/include/opencv2/stitching/stitcher.hpp b/modules/stitching/include/opencv2/stitching/stitcher.hpp index db1bb337d..1616b3d2c 100644 --- a/modules/stitching/include/opencv2/stitching/stitcher.hpp +++ b/modules/stitching/include/opencv2/stitching/stitcher.hpp @@ -51,6 +51,7 @@ #include "detail/exposure_compensate.hpp" #include "detail/seam_finders.hpp" #include "detail/blenders.hpp" +#include "detail/camera.hpp" namespace cv { @@ -78,8 +79,11 @@ public: double panoConfidenceThresh() const { return conf_thresh_; } void setPanoConfidenceThresh(double conf_thresh) { conf_thresh_ = conf_thresh; } - bool horizontalStrightening() const { return horiz_stright_; } - void setHorizontalStrightening(bool flag) { horiz_stright_ = flag; } + bool waveCorrection() const { return do_wave_correct_; } + void setWaveCorrection(bool flag) { do_wave_correct_ = flag; } + + detail::WaveCorrectKind waveCorrectKind() const { return wave_correct_kind_; } + void setWaveCorrectKind(detail::WaveCorrectKind kind) { wave_correct_kind_ = kind; } Ptr featuresFinder() { return features_finder_; } const Ptr featuresFinder() const { return features_finder_; } @@ -96,9 +100,6 @@ public: void setBundleAdjuster(Ptr bundle_adjuster) { bundle_adjuster_ = bundle_adjuster; } - detail::WaveCorrectKind waveCorrectKind() const { return wave_correct_kind_; } - void setWaveCorrectKind(detail::WaveCorrectKind kind) { wave_correct_kind_ = kind; } - Ptr warper() { return warper_; } const Ptr warper() const { return warper_; } void setWarper(Ptr warper) { warper_ = warper; } @@ -119,19 +120,35 @@ public: private: Stitcher() {} + Status matchImages(); + void estimateCameraParams(); + Status composePanorama(cv::Mat &pano); + double registr_resol_; double seam_est_resol_; double compose_resol_; double conf_thresh_; - bool horiz_stright_; Ptr features_finder_; Ptr features_matcher_; Ptr bundle_adjuster_; + bool do_wave_correct_; detail::WaveCorrectKind wave_correct_kind_; Ptr warper_; Ptr exposure_comp_; Ptr seam_finder_; Ptr blender_; + + std::vector imgs_; + std::vector full_img_sizes_; + std::vector features_; + std::vector pairwise_matches_; + std::vector seam_est_imgs_; + std::vector indices_; + std::vector cameras_; + double work_scale_; + double seam_scale_; + double seam_work_aspect_; + double warped_image_scale_; }; } // namespace cv diff --git a/modules/stitching/src/stitcher.cpp b/modules/stitching/src/stitcher.cpp index e946ef17f..cefbacb4f 100644 --- a/modules/stitching/src/stitcher.cpp +++ b/modules/stitching/src/stitcher.cpp @@ -53,10 +53,10 @@ Stitcher Stitcher::createDefault(bool try_use_gpu) stitcher.setSeamEstimationResol(0.1); stitcher.setCompositingResol(ORIG_RESOL); stitcher.setPanoConfidenceThresh(1); - stitcher.setHorizontalStrightening(true); + stitcher.setWaveCorrection(true); + stitcher.setWaveCorrectKind(detail::WAVE_CORRECT_HORIZ); stitcher.setFeaturesMatcher(new detail::BestOf2NearestMatcher(try_use_gpu)); stitcher.setBundleAdjuster(new detail::BundleAdjusterRay()); - stitcher.setWaveCorrectKind(detail::WAVE_CORRECT_HORIZ); #ifndef ANDROID if (try_use_gpu && gpu::getCudaEnabledDeviceCount() > 0) @@ -80,70 +80,83 @@ Stitcher Stitcher::createDefault(bool try_use_gpu) } -Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_) +Stitcher::Status Stitcher::stitch(InputArray imgs, OutputArray pano) { - // TODO split this func - - vector imgs; - imgs_.getMatVector(imgs); - Mat &pano = pano_.getMatRef(); - int64 app_start_time = getTickCount(); - int num_imgs = static_cast(imgs.size()); - if (num_imgs < 2) + imgs.getMatVector(imgs_); + Status status; + + if ((status = matchImages()) != OK) + return status; + + estimateCameraParams(); + + if ((status = composePanorama(pano.getMatRef())) != OK) + return status; + + LOGLN("Finished, total time: " << ((getTickCount() - app_start_time) / getTickFrequency()) << " sec"); + return OK; +} + + +Stitcher::Status Stitcher::matchImages() +{ + if ((int)imgs_.size() < 2) { LOGLN("Need more images"); return ERR_NEED_MORE_IMGS; } - double work_scale = 1, seam_scale = 1, compose_scale = 1; - bool is_work_scale_set = false, is_seam_scale_set = false, is_compose_scale_set = false; + work_scale_ = 1; + seam_work_aspect_ = 1; + seam_scale_ = 1; + bool is_work_scale_set = false; + bool is_seam_scale_set = false; + Mat full_img, img; + features_.resize(imgs_.size()); + seam_est_imgs_.resize(imgs_.size()); + full_img_sizes_.resize(imgs_.size()); LOGLN("Finding features..."); int64 t = getTickCount(); - vector features(num_imgs); - Mat full_img, img; - vector seam_est_imgs(num_imgs); - vector full_img_sizes(num_imgs); - double seam_work_aspect = 1; - - for (int i = 0; i < num_imgs; ++i) + for (size_t i = 0; i < imgs_.size(); ++i) { - full_img = imgs[i]; - full_img_sizes[i] = full_img.size(); + full_img = imgs_[i]; + full_img_sizes_[i] = full_img.size(); if (registr_resol_ < 0) { img = full_img; - work_scale = 1; + work_scale_ = 1; is_work_scale_set = true; } else { if (!is_work_scale_set) { - work_scale = min(1.0, sqrt(registr_resol_ * 1e6 / full_img.size().area())); + work_scale_ = min(1.0, sqrt(registr_resol_ * 1e6 / full_img.size().area())); is_work_scale_set = true; } - resize(full_img, img, Size(), work_scale, work_scale); + resize(full_img, img, Size(), work_scale_, work_scale_); } if (!is_seam_scale_set) { - seam_scale = min(1.0, sqrt(seam_est_resol_ * 1e6 / full_img.size().area())); - seam_work_aspect = seam_scale / work_scale; + seam_scale_ = min(1.0, sqrt(seam_est_resol_ * 1e6 / full_img.size().area())); + seam_work_aspect_ = seam_scale_ / work_scale_; is_seam_scale_set = true; } - (*features_finder_)(img, features[i]); - features[i].img_idx = i; - LOGLN("Features in image #" << i+1 << ": " << features[i].keypoints.size()); + (*features_finder_)(img, features_[i]); + features_[i].img_idx = i; + LOGLN("Features in image #" << i+1 << ": " << features_[i].keypoints.size()); - resize(full_img, img, Size(), seam_scale, seam_scale); - seam_est_imgs[i] = img.clone(); + resize(full_img, img, Size(), seam_scale_, seam_scale_); + seam_est_imgs_[i] = img.clone(); } + // Do it to save memory features_finder_->collectGarbage(); full_img.release(); img.release(); @@ -152,111 +165,120 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_) LOG("Pairwise matching"); t = getTickCount(); - vector pairwise_matches; - (*features_matcher_)(features, pairwise_matches); + (*features_matcher_)(features_, pairwise_matches_); features_matcher_->collectGarbage(); LOGLN("Pairwise matching, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); // Leave only images we are sure are from the same panorama - vector indices = detail::leaveBiggestComponent(features, pairwise_matches, conf_thresh_); + indices_ = detail::leaveBiggestComponent(features_, pairwise_matches_, (float)conf_thresh_); vector seam_est_imgs_subset; vector imgs_subset; vector full_img_sizes_subset; - for (size_t i = 0; i < indices.size(); ++i) + for (size_t i = 0; i < indices_.size(); ++i) { - imgs_subset.push_back(imgs[indices[i]]); - seam_est_imgs_subset.push_back(seam_est_imgs[indices[i]]); - full_img_sizes_subset.push_back(full_img_sizes[indices[i]]); + imgs_subset.push_back(imgs_[indices_[i]]); + seam_est_imgs_subset.push_back(seam_est_imgs_[indices_[i]]); + full_img_sizes_subset.push_back(full_img_sizes_[indices_[i]]); } + seam_est_imgs_ = seam_est_imgs_subset; + imgs_ = imgs_subset; + full_img_sizes_ = full_img_sizes_subset; - seam_est_imgs = seam_est_imgs_subset; - imgs = imgs_subset; - full_img_sizes = full_img_sizes_subset; - - num_imgs = static_cast(imgs.size()); - if (num_imgs < 2) + if ((int)imgs_.size() < 2) { LOGLN("Need more images"); return ERR_NEED_MORE_IMGS; } - vector cameras; - detail::HomographyBasedEstimator estimator; - estimator(features, pairwise_matches, cameras); + return OK; +} - for (size_t i = 0; i < cameras.size(); ++i) + +void Stitcher::estimateCameraParams() +{ + detail::HomographyBasedEstimator estimator; + estimator(features_, pairwise_matches_, cameras_); + + for (size_t i = 0; i < cameras_.size(); ++i) { Mat R; - cameras[i].R.convertTo(R, CV_32F); - cameras[i].R = R; - LOGLN("Initial intrinsic parameters #" << indices[i]+1 << ":\n " << cameras[i].K()); + cameras_[i].R.convertTo(R, CV_32F); + cameras_[i].R = R; + LOGLN("Initial intrinsic parameters #" << indices_[i] + 1 << ":\n " << cameras_[i].K()); } bundle_adjuster_->setConfThresh(conf_thresh_); - (*bundle_adjuster_)(features, pairwise_matches, cameras); + (*bundle_adjuster_)(features_, pairwise_matches_, cameras_); - // Find median focal length + // Find median focal length and use it as final image scale vector focals; - for (size_t i = 0; i < cameras.size(); ++i) + for (size_t i = 0; i < cameras_.size(); ++i) { - LOGLN("Camera #" << indices[i]+1 << ":\n" << cameras[i].K()); - focals.push_back(cameras[i].focal); + LOGLN("Camera #" << indices_[i] + 1 << ":\n" << cameras_[i].K()); + focals.push_back(cameras_[i].focal); } nth_element(focals.begin(), focals.begin() + focals.size()/2, focals.end()); - float warped_image_scale = static_cast(focals[focals.size() / 2]); + warped_image_scale_ = static_cast(focals[focals.size() / 2]); - if (horiz_stright_) + if (do_wave_correct_) { vector rmats; - for (size_t i = 0; i < cameras.size(); ++i) - rmats.push_back(cameras[i].R); + for (size_t i = 0; i < cameras_.size(); ++i) + rmats.push_back(cameras_[i].R); detail::waveCorrect(rmats, wave_correct_kind_); - for (size_t i = 0; i < cameras.size(); ++i) - cameras[i].R = rmats[i]; + for (size_t i = 0; i < cameras_.size(); ++i) + cameras_[i].R = rmats[i]; } +} + +Stitcher::Status Stitcher::composePanorama(Mat &pano) +{ LOGLN("Warping images (auxiliary)... "); - t = getTickCount(); + int64 t = getTickCount(); - vector corners(num_imgs); - vector masks_warped(num_imgs); - vector images_warped(num_imgs); - vector sizes(num_imgs); - vector masks(num_imgs); + vector corners(imgs_.size()); + vector masks_warped(imgs_.size()); + vector images_warped(imgs_.size()); + vector sizes(imgs_.size()); + vector masks(imgs_.size()); - // Preapre images masks - for (int i = 0; i < num_imgs; ++i) + // Prepare image masks + for (size_t i = 0; i < imgs_.size(); ++i) { - masks[i].create(seam_est_imgs[i].size(), CV_8U); + masks[i].create(seam_est_imgs_[i].size(), CV_8U); masks[i].setTo(Scalar::all(255)); } // Warp images and their masks - Ptr warper = warper_->create(warped_image_scale * seam_work_aspect); - for (int i = 0; i < num_imgs; ++i) + Ptr warper = warper_->create(float(warped_image_scale_ * seam_work_aspect_)); + for (size_t i = 0; i < imgs_.size(); ++i) { Mat_ K; - cameras[i].K().convertTo(K, CV_32F); - K(0,0) *= seam_work_aspect; K(0,2) *= seam_work_aspect; - K(1,1) *= seam_work_aspect; K(1,2) *= seam_work_aspect; + cameras_[i].K().convertTo(K, CV_32F); + K(0,0) *= (float)seam_work_aspect_; + K(0,2) *= (float)seam_work_aspect_; + K(1,1) *= (float)seam_work_aspect_; + K(1,2) *= (float)seam_work_aspect_; - corners[i] = warper->warp(seam_est_imgs[i], K, cameras[i].R, images_warped[i], INTER_LINEAR, BORDER_REFLECT); + corners[i] = warper->warp(seam_est_imgs_[i], K, cameras_[i].R, images_warped[i], INTER_LINEAR, BORDER_REFLECT); sizes[i] = images_warped[i].size(); - warper->warp(masks[i], K, cameras[i].R, masks_warped[i], INTER_NEAREST, BORDER_CONSTANT); + warper->warp(masks[i], K, cameras_[i].R, masks_warped[i], INTER_NEAREST, BORDER_CONSTANT); } - vector images_warped_f(num_imgs); - for (int i = 0; i < num_imgs; ++i) + vector images_warped_f(imgs_.size()); + for (size_t i = 0; i < imgs_.size(); ++i) images_warped[i].convertTo(images_warped_f[i], CV_32F); LOGLN("Warping images, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); + // Find seams exposure_comp_->feed(corners, images_warped, masks_warped); seam_finder_->find(images_warped_f, corners, masks_warped); // Release unused memory - seam_est_imgs.clear(); + seam_est_imgs_.clear(); images_warped.clear(); images_warped_f.clear(); masks.clear(); @@ -266,16 +288,21 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_) Mat img_warped, img_warped_s; Mat dilated_mask, seam_mask, mask, mask_warped; + double compose_seam_aspect = 1; double compose_work_aspect = 1; bool is_blender_prepared = false; - for (int img_idx = 0; img_idx < num_imgs; ++img_idx) + double compose_scale = 1; + bool is_compose_scale_set = false; + + Mat full_img, img; + for (size_t img_idx = 0; img_idx < imgs_.size(); ++img_idx) { - LOGLN("Compositing image #" << indices[img_idx]+1); + LOGLN("Compositing image #" << indices_[img_idx] + 1); // Read image and resize it if necessary - full_img = imgs[img_idx]; + full_img = imgs_[img_idx]; if (!is_compose_scale_set) { if (compose_resol_ > 0) @@ -283,32 +310,32 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_) is_compose_scale_set = true; // Compute relative scales - compose_seam_aspect = compose_scale / seam_scale; - compose_work_aspect = compose_scale / work_scale; + compose_seam_aspect = compose_scale / seam_scale_; + compose_work_aspect = compose_scale / work_scale_; // Update warped image scale - warped_image_scale *= static_cast(compose_work_aspect); - warper = warper_->create(warped_image_scale); + warped_image_scale_ *= static_cast(compose_work_aspect); + warper = warper_->create((float)warped_image_scale_); // Update corners and sizes - for (int i = 0; i < num_imgs; ++i) + for (size_t i = 0; i < imgs_.size(); ++i) { // Update intrinsics - cameras[i].focal *= compose_work_aspect; - cameras[i].ppx *= compose_work_aspect; - cameras[i].ppy *= compose_work_aspect; + cameras_[i].focal *= compose_work_aspect; + cameras_[i].ppx *= compose_work_aspect; + cameras_[i].ppy *= compose_work_aspect; // Update corner and size - Size sz = full_img_sizes[i]; + Size sz = full_img_sizes_[i]; if (std::abs(compose_scale - 1) > 1e-1) { - sz.width = cvRound(full_img_sizes[i].width * compose_scale); - sz.height = cvRound(full_img_sizes[i].height * compose_scale); + sz.width = cvRound(full_img_sizes_[i].width * compose_scale); + sz.height = cvRound(full_img_sizes_[i].height * compose_scale); } Mat K; - cameras[i].K().convertTo(K, CV_32F); - Rect roi = warper->warpRoi(sz, K, cameras[i].R); + cameras_[i].K().convertTo(K, CV_32F); + Rect roi = warper->warpRoi(sz, K, cameras_[i].R); corners[i] = roi.tl(); sizes[i] = roi.size(); } @@ -321,15 +348,15 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_) Size img_size = img.size(); Mat K; - cameras[img_idx].K().convertTo(K, CV_32F); + cameras_[img_idx].K().convertTo(K, CV_32F); // Warp the current image - warper->warp(img, K, cameras[img_idx].R, img_warped, INTER_LINEAR, BORDER_REFLECT); + warper->warp(img, K, cameras_[img_idx].R, img_warped, INTER_LINEAR, BORDER_REFLECT); // Warp the current image mask mask.create(img_size, CV_8U); mask.setTo(Scalar::all(255)); - warper->warp(mask, K, cameras[img_idx].R, mask_warped, INTER_NEAREST, BORDER_CONSTANT); + warper->warp(mask, K, cameras_[img_idx].R, mask_warped, INTER_NEAREST, BORDER_CONSTANT); // Compensate exposure exposure_comp_->apply(img_idx, corners[img_idx], img_warped, mask_warped); @@ -339,8 +366,10 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_) img.release(); mask.release(); + // Make sure seam mask has proper size dilate(masks_warped[img_idx], dilated_mask, Mat()); resize(dilated_mask, seam_mask, mask_warped.size()); + mask_warped = seam_mask & mask_warped; if (!is_blender_prepared) @@ -362,8 +391,6 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_) // so convert it to avoid user confusing result.convertTo(pano, CV_8U); - LOGLN("Finished, total time: " << ((getTickCount() - app_start_time) / getTickFrequency()) << " sec"); - return OK; }