generic-library/opencv
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Refactored stitching module

Browse Source
This commit is contained in:
Alexey Spizhevoy 2011-09-26 08:52:31 +00:00
parent 67a9b79433
commit 551113292b
2 changed files with 153 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
modules/stitching/include/opencv2/stitching/stitcher.hpp
View File

@ -51,6 +51,7 @@
#include "detail/exposure_compensate.hpp"
#include "detail/seam_finders.hpp"
#include "detail/blenders.hpp"
#include "detail/camera.hpp"
namespace cv {
@ -78,8 +79,11 @@ public:
double panoConfidenceThresh() const { return conf_thresh_; }
void setPanoConfidenceThresh(double conf_thresh) { conf_thresh_ = conf_thresh; }
bool horizontalStrightening() const { return horiz_stright_; }
void setHorizontalStrightening(bool flag) { horiz_stright_ = flag; }
bool waveCorrection() const { return do_wave_correct_; }
void setWaveCorrection(bool flag) { do_wave_correct_ = flag; }
detail::WaveCorrectKind waveCorrectKind() const { return wave_correct_kind_; }
void setWaveCorrectKind(detail::WaveCorrectKind kind) { wave_correct_kind_ = kind; }
Ptr<detail::FeaturesFinder> featuresFinder() { return features_finder_; }
const Ptr<detail::FeaturesFinder> featuresFinder() const { return features_finder_; }
@ -96,9 +100,6 @@ public:
void setBundleAdjuster(Ptr<detail::BundleAdjusterBase> bundle_adjuster)
{ bundle_adjuster_ = bundle_adjuster; }
detail::WaveCorrectKind waveCorrectKind() const { return wave_correct_kind_; }
void setWaveCorrectKind(detail::WaveCorrectKind kind) { wave_correct_kind_ = kind; }
Ptr<WarperCreator> warper() { return warper_; }
const Ptr<WarperCreator> warper() const { return warper_; }
void setWarper(Ptr<WarperCreator> warper) { warper_ = warper; }
@ -119,19 +120,35 @@ public:
private:
Stitcher() {}
Status matchImages();
void estimateCameraParams();
Status composePanorama(cv::Mat &pano);
double registr_resol_;
double seam_est_resol_;
double compose_resol_;
double conf_thresh_;
bool horiz_stright_;
Ptr<detail::FeaturesFinder> features_finder_;
Ptr<detail::FeaturesMatcher> features_matcher_;
Ptr<detail::BundleAdjusterBase> bundle_adjuster_;
bool do_wave_correct_;
detail::WaveCorrectKind wave_correct_kind_;
Ptr<WarperCreator> warper_;
Ptr<detail::ExposureCompensator> exposure_comp_;
Ptr<detail::SeamFinder> seam_finder_;
Ptr<detail::Blender> blender_;
std::vector<cv::Mat> imgs_;
std::vector<cv::Size> full_img_sizes_;
std::vector<detail::ImageFeatures> features_;
std::vector<detail::MatchesInfo> pairwise_matches_;
std::vector<cv::Mat> seam_est_imgs_;
std::vector<int> indices_;
std::vector<detail::CameraParams> cameras_;
double work_scale_;
double seam_scale_;
double seam_work_aspect_;
double warped_image_scale_;
};
} // namespace cv

233
modules/stitching/src/stitcher.cpp
View File

@ -53,10 +53,10 @@ Stitcher Stitcher::createDefault(bool try_use_gpu)
stitcher.setSeamEstimationResol(0.1);
stitcher.setCompositingResol(ORIG_RESOL);
stitcher.setPanoConfidenceThresh(1);
stitcher.setHorizontalStrightening(true);
stitcher.setWaveCorrection(true);
stitcher.setWaveCorrectKind(detail::WAVE_CORRECT_HORIZ);
stitcher.setFeaturesMatcher(new detail::BestOf2NearestMatcher(try_use_gpu));
stitcher.setBundleAdjuster(new detail::BundleAdjusterRay());
stitcher.setWaveCorrectKind(detail::WAVE_CORRECT_HORIZ);
#ifndef ANDROID
if (try_use_gpu && gpu::getCudaEnabledDeviceCount() > 0)
@ -80,70 +80,83 @@ Stitcher Stitcher::createDefault(bool try_use_gpu)
}
Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_)
Stitcher::Status Stitcher::stitch(InputArray imgs, OutputArray pano)
{
// TODO split this func
vector<Mat> imgs;
imgs_.getMatVector(imgs);
Mat &pano = pano_.getMatRef();
int64 app_start_time = getTickCount();
int num_imgs = static_cast<int>(imgs.size());
if (num_imgs < 2)
imgs.getMatVector(imgs_);
Status status;
if ((status = matchImages()) != OK)
return status;
estimateCameraParams();
if ((status = composePanorama(pano.getMatRef())) != OK)
return status;
LOGLN("Finished, total time: " << ((getTickCount() - app_start_time) / getTickFrequency()) << " sec");
return OK;
}
Stitcher::Status Stitcher::matchImages()
{
if ((int)imgs_.size() < 2)
{
LOGLN("Need more images");
return ERR_NEED_MORE_IMGS;
}
double work_scale = 1, seam_scale = 1, compose_scale = 1;
bool is_work_scale_set = false, is_seam_scale_set = false, is_compose_scale_set = false;
work_scale_ = 1;
seam_work_aspect_ = 1;
seam_scale_ = 1;
bool is_work_scale_set = false;
bool is_seam_scale_set = false;
Mat full_img, img;
features_.resize(imgs_.size());
seam_est_imgs_.resize(imgs_.size());
full_img_sizes_.resize(imgs_.size());
LOGLN("Finding features...");
int64 t = getTickCount();
vector<detail::ImageFeatures> features(num_imgs);
Mat full_img, img;
vector<Mat> seam_est_imgs(num_imgs);
vector<Size> full_img_sizes(num_imgs);
double seam_work_aspect = 1;
for (int i = 0; i < num_imgs; ++i)
for (size_t i = 0; i < imgs_.size(); ++i)
{
full_img = imgs[i];
full_img_sizes[i] = full_img.size();
full_img = imgs_[i];
full_img_sizes_[i] = full_img.size();
if (registr_resol_ < 0)
{
img = full_img;
work_scale = 1;
work_scale_ = 1;
is_work_scale_set = true;
}
else
{
if (!is_work_scale_set)
{
work_scale = min(1.0, sqrt(registr_resol_ * 1e6 / full_img.size().area()));
work_scale_ = min(1.0, sqrt(registr_resol_ * 1e6 / full_img.size().area()));
is_work_scale_set = true;
}
resize(full_img, img, Size(), work_scale, work_scale);
resize(full_img, img, Size(), work_scale_, work_scale_);
}
if (!is_seam_scale_set)
{
seam_scale = min(1.0, sqrt(seam_est_resol_ * 1e6 / full_img.size().area()));
seam_work_aspect = seam_scale / work_scale;
seam_scale_ = min(1.0, sqrt(seam_est_resol_ * 1e6 / full_img.size().area()));
seam_work_aspect_ = seam_scale_ / work_scale_;
is_seam_scale_set = true;
}
(*features_finder_)(img, features[i]);
features[i].img_idx = i;
LOGLN("Features in image #" << i+1 << ": " << features[i].keypoints.size());
(*features_finder_)(img, features_[i]);
features_[i].img_idx = i;
LOGLN("Features in image #" << i+1 << ": " << features_[i].keypoints.size());
resize(full_img, img, Size(), seam_scale, seam_scale);
seam_est_imgs[i] = img.clone();
resize(full_img, img, Size(), seam_scale_, seam_scale_);
seam_est_imgs_[i] = img.clone();
}
// Do it to save memory
features_finder_->collectGarbage();
full_img.release();
img.release();
@ -152,111 +165,120 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_)
LOG("Pairwise matching");
t = getTickCount();
vector<detail::MatchesInfo> pairwise_matches;
(*features_matcher_)(features, pairwise_matches);
(*features_matcher_)(features_, pairwise_matches_);
features_matcher_->collectGarbage();
LOGLN("Pairwise matching, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec");
// Leave only images we are sure are from the same panorama
vector<int> indices = detail::leaveBiggestComponent(features, pairwise_matches, conf_thresh_);
indices_ = detail::leaveBiggestComponent(features_, pairwise_matches_, (float)conf_thresh_);
vector<Mat> seam_est_imgs_subset;
vector<Mat> imgs_subset;
vector<Size> full_img_sizes_subset;
for (size_t i = 0; i < indices.size(); ++i)
for (size_t i = 0; i < indices_.size(); ++i)
{
imgs_subset.push_back(imgs[indices[i]]);
seam_est_imgs_subset.push_back(seam_est_imgs[indices[i]]);
full_img_sizes_subset.push_back(full_img_sizes[indices[i]]);
imgs_subset.push_back(imgs_[indices_[i]]);
seam_est_imgs_subset.push_back(seam_est_imgs_[indices_[i]]);
full_img_sizes_subset.push_back(full_img_sizes_[indices_[i]]);
}
seam_est_imgs_ = seam_est_imgs_subset;
imgs_ = imgs_subset;
full_img_sizes_ = full_img_sizes_subset;
seam_est_imgs = seam_est_imgs_subset;
imgs = imgs_subset;
full_img_sizes = full_img_sizes_subset;
num_imgs = static_cast<int>(imgs.size());
if (num_imgs < 2)
if ((int)imgs_.size() < 2)
{
LOGLN("Need more images");
return ERR_NEED_MORE_IMGS;
}
vector<detail::CameraParams> cameras;
detail::HomographyBasedEstimator estimator;
estimator(features, pairwise_matches, cameras);
return OK;
}
for (size_t i = 0; i < cameras.size(); ++i)
void Stitcher::estimateCameraParams()
{
detail::HomographyBasedEstimator estimator;
estimator(features_, pairwise_matches_, cameras_);
for (size_t i = 0; i < cameras_.size(); ++i)
{
Mat R;
cameras[i].R.convertTo(R, CV_32F);
cameras[i].R = R;
LOGLN("Initial intrinsic parameters #" << indices[i]+1 << ":\n " << cameras[i].K());
cameras_[i].R.convertTo(R, CV_32F);
cameras_[i].R = R;
LOGLN("Initial intrinsic parameters #" << indices_[i] + 1 << ":\n " << cameras_[i].K());
}
bundle_adjuster_->setConfThresh(conf_thresh_);
(*bundle_adjuster_)(features, pairwise_matches, cameras);
(*bundle_adjuster_)(features_, pairwise_matches_, cameras_);
// Find median focal length
// Find median focal length and use it as final image scale
vector<double> focals;
for (size_t i = 0; i < cameras.size(); ++i)
for (size_t i = 0; i < cameras_.size(); ++i)
{
LOGLN("Camera #" << indices[i]+1 << ":\n" << cameras[i].K());
focals.push_back(cameras[i].focal);
LOGLN("Camera #" << indices_[i] + 1 << ":\n" << cameras_[i].K());
focals.push_back(cameras_[i].focal);
}
nth_element(focals.begin(), focals.begin() + focals.size()/2, focals.end());
float warped_image_scale = static_cast<float>(focals[focals.size() / 2]);
warped_image_scale_ = static_cast<float>(focals[focals.size() / 2]);
if (horiz_stright_)
if (do_wave_correct_)
{
vector<Mat> rmats;
for (size_t i = 0; i < cameras.size(); ++i)
rmats.push_back(cameras[i].R);
for (size_t i = 0; i < cameras_.size(); ++i)
rmats.push_back(cameras_[i].R);
detail::waveCorrect(rmats, wave_correct_kind_);
for (size_t i = 0; i < cameras.size(); ++i)
cameras[i].R = rmats[i];
for (size_t i = 0; i < cameras_.size(); ++i)
cameras_[i].R = rmats[i];
}
}
Stitcher::Status Stitcher::composePanorama(Mat &pano)
{
LOGLN("Warping images (auxiliary)... ");
t = getTickCount();
int64 t = getTickCount();
vector<Point> corners(num_imgs);
vector<Mat> masks_warped(num_imgs);
vector<Mat> images_warped(num_imgs);
vector<Size> sizes(num_imgs);
vector<Mat> masks(num_imgs);
vector<Point> corners(imgs_.size());
vector<Mat> masks_warped(imgs_.size());
vector<Mat> images_warped(imgs_.size());
vector<Size> sizes(imgs_.size());
vector<Mat> masks(imgs_.size());
// Preapre images masks
for (int i = 0; i < num_imgs; ++i)
// Prepare image masks
for (size_t i = 0; i < imgs_.size(); ++i)
{
masks[i].create(seam_est_imgs[i].size(), CV_8U);
masks[i].create(seam_est_imgs_[i].size(), CV_8U);
masks[i].setTo(Scalar::all(255));
}
// Warp images and their masks
Ptr<detail::Warper> warper = warper_->create(warped_image_scale * seam_work_aspect);
for (int i = 0; i < num_imgs; ++i)
Ptr<detail::Warper> warper = warper_->create(float(warped_image_scale_ * seam_work_aspect_));
for (size_t i = 0; i < imgs_.size(); ++i)
{
Mat_<float> K;
cameras[i].K().convertTo(K, CV_32F);
K(0,0) *= seam_work_aspect; K(0,2) *= seam_work_aspect;
K(1,1) *= seam_work_aspect; K(1,2) *= seam_work_aspect;
cameras_[i].K().convertTo(K, CV_32F);
K(0,0) *= (float)seam_work_aspect_;
K(0,2) *= (float)seam_work_aspect_;
K(1,1) *= (float)seam_work_aspect_;
K(1,2) *= (float)seam_work_aspect_;
corners[i] = warper->warp(seam_est_imgs[i], K, cameras[i].R, images_warped[i], INTER_LINEAR, BORDER_REFLECT);
corners[i] = warper->warp(seam_est_imgs_[i], K, cameras_[i].R, images_warped[i], INTER_LINEAR, BORDER_REFLECT);
sizes[i] = images_warped[i].size();
warper->warp(masks[i], K, cameras[i].R, masks_warped[i], INTER_NEAREST, BORDER_CONSTANT);
warper->warp(masks[i], K, cameras_[i].R, masks_warped[i], INTER_NEAREST, BORDER_CONSTANT);
}
vector<Mat> images_warped_f(num_imgs);
for (int i = 0; i < num_imgs; ++i)
vector<Mat> images_warped_f(imgs_.size());
for (size_t i = 0; i < imgs_.size(); ++i)
images_warped[i].convertTo(images_warped_f[i], CV_32F);
LOGLN("Warping images, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec");
// Find seams
exposure_comp_->feed(corners, images_warped, masks_warped);
seam_finder_->find(images_warped_f, corners, masks_warped);
// Release unused memory
seam_est_imgs.clear();
seam_est_imgs_.clear();
images_warped.clear();
images_warped_f.clear();
masks.clear();
@ -266,16 +288,21 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_)
Mat img_warped, img_warped_s;
Mat dilated_mask, seam_mask, mask, mask_warped;
double compose_seam_aspect = 1;
double compose_work_aspect = 1;
bool is_blender_prepared = false;
for (int img_idx = 0; img_idx < num_imgs; ++img_idx)
double compose_scale = 1;
bool is_compose_scale_set = false;
Mat full_img, img;
for (size_t img_idx = 0; img_idx < imgs_.size(); ++img_idx)
{
LOGLN("Compositing image #" << indices[img_idx]+1);
LOGLN("Compositing image #" << indices_[img_idx] + 1);
// Read image and resize it if necessary
full_img = imgs[img_idx];
full_img = imgs_[img_idx];
if (!is_compose_scale_set)
{
if (compose_resol_ > 0)
@ -283,32 +310,32 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_)
is_compose_scale_set = true;
// Compute relative scales
compose_seam_aspect = compose_scale / seam_scale;
compose_work_aspect = compose_scale / work_scale;
compose_seam_aspect = compose_scale / seam_scale_;
compose_work_aspect = compose_scale / work_scale_;
// Update warped image scale
warped_image_scale *= static_cast<float>(compose_work_aspect);
warper = warper_->create(warped_image_scale);
warped_image_scale_ *= static_cast<float>(compose_work_aspect);
warper = warper_->create((float)warped_image_scale_);
// Update corners and sizes
for (int i = 0; i < num_imgs; ++i)
for (size_t i = 0; i < imgs_.size(); ++i)
{
// Update intrinsics
cameras[i].focal *= compose_work_aspect;
cameras[i].ppx *= compose_work_aspect;
cameras[i].ppy *= compose_work_aspect;
cameras_[i].focal *= compose_work_aspect;
cameras_[i].ppx *= compose_work_aspect;
cameras_[i].ppy *= compose_work_aspect;
// Update corner and size
Size sz = full_img_sizes[i];
Size sz = full_img_sizes_[i];
if (std::abs(compose_scale - 1) > 1e-1)
{
sz.width = cvRound(full_img_sizes[i].width * compose_scale);
sz.height = cvRound(full_img_sizes[i].height * compose_scale);
sz.width = cvRound(full_img_sizes_[i].width * compose_scale);
sz.height = cvRound(full_img_sizes_[i].height * compose_scale);
}
Mat K;
cameras[i].K().convertTo(K, CV_32F);
Rect roi = warper->warpRoi(sz, K, cameras[i].R);
cameras_[i].K().convertTo(K, CV_32F);
Rect roi = warper->warpRoi(sz, K, cameras_[i].R);
corners[i] = roi.tl();
sizes[i] = roi.size();
}
@ -321,15 +348,15 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_)
Size img_size = img.size();
Mat K;
cameras[img_idx].K().convertTo(K, CV_32F);
cameras_[img_idx].K().convertTo(K, CV_32F);
// Warp the current image
warper->warp(img, K, cameras[img_idx].R, img_warped, INTER_LINEAR, BORDER_REFLECT);
warper->warp(img, K, cameras_[img_idx].R, img_warped, INTER_LINEAR, BORDER_REFLECT);
// Warp the current image mask
mask.create(img_size, CV_8U);
mask.setTo(Scalar::all(255));
warper->warp(mask, K, cameras[img_idx].R, mask_warped, INTER_NEAREST, BORDER_CONSTANT);
warper->warp(mask, K, cameras_[img_idx].R, mask_warped, INTER_NEAREST, BORDER_CONSTANT);
// Compensate exposure
exposure_comp_->apply(img_idx, corners[img_idx], img_warped, mask_warped);
@ -339,8 +366,10 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_)
img.release();
mask.release();
// Make sure seam mask has proper size
dilate(masks_warped[img_idx], dilated_mask, Mat());
resize(dilated_mask, seam_mask, mask_warped.size());
mask_warped = seam_mask & mask_warped;
if (!is_blender_prepared)
@ -362,8 +391,6 @@ Stitcher::Status Stitcher::stitch(InputArray imgs_, OutputArray pano_)
// so convert it to avoid user confusing
result.convertTo(pano, CV_8U);
LOGLN("Finished, total time: " << ((getTickCount() - app_start_time) / getTickFrequency()) << " sec");
return OK;
}