From 16e6c45ed76e5b3987e80da9a12b8bd77dd5b2c8 Mon Sep 17 00:00:00 2001 From: Alexey Spizhevoy Date: Sat, 21 May 2011 11:09:47 +0000 Subject: [PATCH] changes blenders interface in opencv_stitching --- modules/stitching/blenders.cpp | 336 +++++++++++++++------------------ modules/stitching/blenders.hpp | 45 ++--- modules/stitching/main.cpp | 51 +++-- 3 files changed, 204 insertions(+), 228 deletions(-) diff --git a/modules/stitching/blenders.cpp b/modules/stitching/blenders.cpp index 04cc7b98a..ae95ad108 100644 --- a/modules/stitching/blenders.cpp +++ b/modules/stitching/blenders.cpp @@ -21,249 +21,209 @@ Ptr Blender::createDefault(int type) } -Point Blender::operator ()(const vector &src, const vector &corners, const vector &masks, - Mat& dst) +void Blender::prepare(const vector &corners, const vector &sizes) { - Mat dst_mask; - return (*this)(src, corners, masks, dst, dst_mask); + prepare(resultRoi(corners, sizes)); } -Point Blender::operator ()(const vector &src, const vector &corners, const vector &masks, - Mat &dst, Mat &dst_mask) +void Blender::prepare(Rect dst_roi) { - Point dst_tl = blend(src, corners, masks, dst, dst_mask); - dst.setTo(Scalar::all(0), dst_mask == 0); - return dst_tl; + dst_.create(dst_roi.size(), CV_32FC3); + dst_.setTo(Scalar::all(0)); + dst_mask_.create(dst_roi.size(), CV_8U); + dst_mask_.setTo(Scalar::all(0)); + dst_roi_ = dst_roi; } -Point Blender::blend(const vector &src, const vector &corners, const vector &masks, - Mat &dst, Mat &dst_mask) +void Blender::feed(const Mat &img, const Mat &mask, Point tl) { - for (size_t i = 0; i < src.size(); ++i) + CV_Assert(img.type() == CV_32FC3); + CV_Assert(mask.type() == CV_8U); + + int dx = tl.x - dst_roi_.x; + int dy = tl.y - dst_roi_.y; + + for (int y = 0; y < img.rows; ++y) { - CV_Assert(src[i].type() == CV_32FC3); - CV_Assert(masks[i].type() == CV_8U); - } - const int image_type = src[0].type(); + const Point3f *src_row = img.ptr(y); + Point3f *dst_row = dst_.ptr(dy + y); - Rect dst_roi = resultRoi(src, corners); + const uchar *mask_row = mask.ptr(y); + uchar *dst_mask_row = dst_mask_.ptr(dy + y); - dst.create(dst_roi.size(), image_type); - dst.setTo(Scalar::all(0)); - - dst_mask.create(dst_roi.size(), CV_8U); - dst_mask.setTo(Scalar::all(0)); - - for (size_t i = 0; i < src.size(); ++i) - { - int dx = corners[i].x - dst_roi.x; - int dy = corners[i].y - dst_roi.y; - - for (int y = 0; y < src[i].rows; ++y) + for (int x = 0; x < img.cols; ++x) { - const Point3f *src_row = src[i].ptr(y); - Point3f *dst_row = dst.ptr(dy + y); - - const uchar *mask_row = masks[i].ptr(y); - uchar *dst_mask_row = dst_mask.ptr(dy + y); - - for (int x = 0; x < src[i].cols; ++x) - { - if (mask_row[x]) - dst_row[dx + x] = src_row[x]; - dst_mask_row[dx + x] |= mask_row[x]; - } + if (mask_row[x]) + dst_row[dx + x] = src_row[x]; + dst_mask_row[dx + x] |= mask_row[x]; } } - - return dst_roi.tl(); } -Point FeatherBlender::blend(const vector &src, const vector &corners, const vector &masks, - Mat &dst, Mat &dst_mask) +void Blender::blend(Mat &dst, Mat &dst_mask) { - vector weights(masks.size()); - for (size_t i = 0; i < weights.size(); ++i) - createWeightMap(masks[i], sharpness_, weights[i]); - - Mat dst_weight; - Point dst_tl = blendLinear(src, corners, weights, dst, dst_weight); - dst_mask = dst_weight > WEIGHT_EPS; - - return dst_tl; + dst_.setTo(Scalar::all(0), dst_mask_ == 0); + dst = dst_; + dst_mask = dst_mask_; + dst_.release(); + dst_mask_.release(); } -Point MultiBandBlender::blend(const vector &src, const vector &corners, const vector &masks, - Mat &dst, Mat &dst_mask) +void FeatherBlender::prepare(Rect dst_roi) { - CV_Assert(src.size() == corners.size() && src.size() == masks.size()); - const int num_images = src.size(); - const int img_type = src[0].type(); + Blender::prepare(dst_roi); + dst_weight_map_.create(dst_roi.size(), CV_32F); + dst_weight_map_.setTo(0); +} - Rect dst_roi = resultRoi(src, corners); - computeResultMask(masks, corners, dst_mask); - vector dst_pyr_laplace(num_bands_ + 1); - dst_pyr_laplace[0].create(dst_roi.size(), img_type); - dst_pyr_laplace[0].setTo(Scalar::all(0)); +void FeatherBlender::feed(const Mat &img, const Mat &mask, Point tl) +{ + CV_Assert(img.type() == CV_32FC3); + CV_Assert(mask.type() == CV_8U); - vector dst_band_weights(num_bands_ + 1); - dst_band_weights[0].create(dst_roi.size(), CV_32F); - dst_band_weights[0].setTo(0); + int dx = tl.x - dst_roi_.x; + int dy = tl.y - dst_roi_.y; + + createWeightMap(mask, sharpness_, weight_map_); + + for (int y = 0; y < img.rows; ++y) + { + const Point3f* src_row = img.ptr(y); + Point3f* dst_row = dst_.ptr(dy + y); + + const float* weight_row = weight_map_.ptr(y); + float* dst_weight_row = dst_weight_map_.ptr(dy + y); + + for (int x = 0; x < img.cols; ++x) + { + dst_row[dx + x] += src_row[x] * weight_row[x]; + dst_weight_row[dx + x] += weight_row[x]; + } + } +} + + +void FeatherBlender::blend(Mat &dst, Mat &dst_mask) +{ + normalize(dst_weight_map_, dst_); + dst_mask_ = dst_weight_map_ > WEIGHT_EPS; + Blender::blend(dst, dst_mask); +} + + +void MultiBandBlender::prepare(Rect dst_roi) +{ + Blender::prepare(dst_roi); + + dst_pyr_laplace_.resize(num_bands_ + 1); + dst_pyr_laplace_[0].create(dst_roi.size(), CV_32FC3); + dst_pyr_laplace_[0].setTo(Scalar::all(0)); + + dst_band_weights_.resize(num_bands_ + 1); + dst_band_weights_[0].create(dst_roi.size(), CV_32F); + dst_band_weights_[0].setTo(0); for (int i = 1; i <= num_bands_; ++i) { - dst_pyr_laplace[i].create((dst_pyr_laplace[i - 1].rows + 1) / 2, - (dst_pyr_laplace[i - 1].cols + 1) / 2, img_type); - dst_pyr_laplace[i].setTo(Scalar::all(0)); - - dst_band_weights[i].create((dst_band_weights[i - 1].rows + 1) / 2, - (dst_band_weights[i - 1].cols + 1) / 2, CV_32F); - dst_band_weights[i].setTo(0); + dst_pyr_laplace_[i].create((dst_pyr_laplace_[i - 1].rows + 1) / 2, + (dst_pyr_laplace_[i - 1].cols + 1) / 2, CV_32FC3); + dst_band_weights_[i].create((dst_band_weights_[i - 1].rows + 1) / 2, + (dst_band_weights_[i - 1].cols + 1) / 2, CV_32F); + dst_pyr_laplace_[i].setTo(Scalar::all(0)); + dst_band_weights_[i].setTo(0); } +} - for (int img_idx = 0; img_idx < num_images; ++img_idx) + +void MultiBandBlender::feed(const Mat &img, const Mat &mask, Point tl) +{ + CV_Assert(img.type() == CV_32FC3); + CV_Assert(mask.type() == CV_8U); + + int top = tl.y - dst_roi_.y; + int left = tl.x - dst_roi_.x; + int bottom = dst_roi_.br().y - tl.y - img.rows; + int right = dst_roi_.br().x - tl.x - img.cols; + + // Create the source image Laplacian pyramid + vector src_pyr_gauss(num_bands_ + 1); + copyMakeBorder(img, src_pyr_gauss[0], top, bottom, left, right, + BORDER_REFLECT); + for (int i = 0; i < num_bands_; ++i) + pyrDown(src_pyr_gauss[i], src_pyr_gauss[i + 1]); + vector src_pyr_laplace; + createLaplacePyr(src_pyr_gauss, src_pyr_laplace); + src_pyr_gauss.clear(); + + // Create the weight map Gaussian pyramid + Mat weight_map; + mask.convertTo(weight_map, CV_32F, 1./255.); + vector weight_pyr_gauss(num_bands_ + 1); + copyMakeBorder(weight_map, weight_pyr_gauss[0], top, bottom, left, right, + BORDER_CONSTANT); + for (int i = 0; i < num_bands_; ++i) + pyrDown(weight_pyr_gauss[i], weight_pyr_gauss[i + 1]); + + // Add weighted layer of the source image to the final Laplacian pyramid layer + for (int i = 0; i <= num_bands_; ++i) { - int top = corners[img_idx].y - dst_roi.y; - int bottom = dst_roi.br().y - corners[img_idx].y - src[img_idx].rows; - int left = corners[img_idx].x - dst_roi.x; - int right = dst_roi.br().x - corners[img_idx].x - src[img_idx].cols; - - vector src_pyr_gauss(num_bands_ + 1); - copyMakeBorder(src[img_idx], src_pyr_gauss[0], top, bottom, left, right, BORDER_REFLECT); - for (int i = 0; i < num_bands_; ++i) - pyrDown(src_pyr_gauss[i], src_pyr_gauss[i + 1]); - - vector src_pyr_laplace; - createLaplacePyr(src_pyr_gauss, src_pyr_laplace); - - vector weight_pyr_gauss(num_bands_ + 1); - Mat mask_f; - masks[img_idx].convertTo(mask_f, CV_32F, 1./255.); - copyMakeBorder(mask_f, weight_pyr_gauss[0], top, bottom, left, right, BORDER_CONSTANT); - for (int i = 0; i < num_bands_; ++i) - pyrDown(weight_pyr_gauss[i], weight_pyr_gauss[i + 1]); - - for (int band_idx = 0; band_idx <= num_bands_; ++band_idx) + for (int y = 0; y < dst_pyr_laplace_[i].rows; ++y) { - for (int y = 0; y < dst_pyr_laplace[band_idx].rows; ++y) - { - const Point3f* src_row = src_pyr_laplace[band_idx].ptr(y); - const float* weight_row = weight_pyr_gauss[band_idx].ptr(y); - Point3f* dst_row = dst_pyr_laplace[band_idx].ptr(y); - for (int x = 0; x < dst_pyr_laplace[band_idx].cols; ++x) - dst_row[x] += src_row[x] * weight_row[x]; - } - dst_band_weights[band_idx] += weight_pyr_gauss[band_idx]; + const Point3f* src_row = src_pyr_laplace[i].ptr(y); + Point3f* dst_row = dst_pyr_laplace_[i].ptr(y); + + const float* weight_row = weight_pyr_gauss[i].ptr(y); + + for (int x = 0; x < dst_pyr_laplace_[i].cols; ++x) + dst_row[x] += src_row[x] * weight_row[x]; } - } + dst_band_weights_[i] += weight_pyr_gauss[i]; + } +} - for (int band_idx = 0; band_idx <= num_bands_; ++band_idx) - normalize(dst_band_weights[band_idx], dst_pyr_laplace[band_idx]); - restoreImageFromLaplacePyr(dst_pyr_laplace); - dst = dst_pyr_laplace[0]; - return dst_roi.tl(); +void MultiBandBlender::blend(Mat &dst, Mat &dst_mask) +{ + for (int i = 0; i <= num_bands_; ++i) + normalize(dst_band_weights_[i], dst_pyr_laplace_[i]); + + restoreImageFromLaplacePyr(dst_pyr_laplace_); + + dst_ = dst_pyr_laplace_[0]; + dst_mask_ = dst_band_weights_[0] > WEIGHT_EPS; + dst_pyr_laplace_.clear(); + dst_band_weights_.clear(); + + Blender::blend(dst, dst_mask); } ////////////////////////////////////////////////////////////////////////////// // Auxiliary functions -Rect resultRoi(const vector &src, const vector &corners) +Rect resultRoi(const vector &corners, const vector &sizes) { Point tl(numeric_limits::max(), numeric_limits::max()); Point br(numeric_limits::min(), numeric_limits::min()); - CV_Assert(src.size() == corners.size()); - for (size_t i = 0; i < src.size(); ++i) + CV_Assert(sizes.size() == corners.size()); + for (size_t i = 0; i < corners.size(); ++i) { tl.x = min(tl.x, corners[i].x); tl.y = min(tl.y, corners[i].y); - br.x = max(br.x, corners[i].x + src[i].cols); - br.y = max(br.y, corners[i].y + src[i].rows); + br.x = max(br.x, corners[i].x + sizes[i].width); + br.y = max(br.y, corners[i].y + sizes[i].height); } return Rect(tl, br); } -Point computeResultMask(const vector &masks, const vector &corners, Mat &dst_mask) -{ - Rect dst_roi = resultRoi(masks, corners); - - dst_mask.create(dst_roi.size(), CV_8U); - dst_mask.setTo(Scalar::all(0)); - - for (size_t i = 0; i < masks.size(); ++i) - { - int dx = corners[i].x - dst_roi.x; - int dy = corners[i].y - dst_roi.y; - - for (int y = 0; y < masks[i].rows; ++y) - { - const uchar *mask_row = masks[i].ptr(y); - uchar *dst_mask_row = dst_mask.ptr(dy + y); - - for (int x = 0; x < masks[i].cols; ++x) - dst_mask_row[dx + x] |= mask_row[x]; - } - } - - return dst_roi.tl(); -} - - -Point blendLinear(const vector &src, const vector &corners, const vector &weights, - Mat &dst, Mat& dst_weight) -{ - for (size_t i = 0; i < src.size(); ++i) - { - CV_Assert(src[i].type() == CV_32FC3); - CV_Assert(weights[i].type() == CV_32F); - } - const int image_type = src[0].type(); - - Rect dst_roi = resultRoi(src, corners); - - dst.create(dst_roi.size(), image_type); - dst.setTo(Scalar::all(0)); - - dst_weight.create(dst_roi.size(), CV_32F); - dst_weight.setTo(Scalar::all(0)); - - // Compute colors sums and weights - for (size_t i = 0; i < src.size(); ++i) - { - int dx = corners[i].x - dst_roi.x; - int dy = corners[i].y - dst_roi.y; - - for (int y = 0; y < src[i].rows; ++y) - { - const Point3f *src_row = src[i].ptr(y); - Point3f *dst_row = dst.ptr(dy + y); - - const float *weight_row = weights[i].ptr(y); - float *dst_weight_row = dst_weight.ptr(dy + y); - - for (int x = 0; x < src[i].cols; ++x) - { - dst_row[dx + x] += src_row[x] * weight_row[x]; - dst_weight_row[dx + x] += weight_row[x]; - } - } - } - - normalize(dst_weight, dst); - - return dst_roi.tl(); -} - - void normalize(const Mat& weight, Mat& src) { CV_Assert(weight.type() == CV_32F); diff --git a/modules/stitching/blenders.hpp b/modules/stitching/blenders.hpp index 308a81f8c..a3942a353 100644 --- a/modules/stitching/blenders.hpp +++ b/modules/stitching/blenders.hpp @@ -9,58 +9,59 @@ class Blender { public: enum { NO, FEATHER, MULTI_BAND }; - static cv::Ptr createDefault(int type); - cv::Point operator ()(const std::vector &src, const std::vector &corners, const std::vector &masks, - cv::Mat& dst); - cv::Point operator ()(const std::vector &src, const std::vector &corners, const std::vector &masks, - cv::Mat& dst, cv::Mat& dst_mask); + void prepare(const std::vector &corners, const std::vector &sizes); + virtual void prepare(cv::Rect dst_roi); + virtual void feed(const cv::Mat &img, const cv::Mat &mask, cv::Point tl); + virtual void blend(cv::Mat &dst, cv::Mat &dst_mask); protected: - virtual cv::Point blend(const std::vector &src, const std::vector &corners, const std::vector &masks, - cv::Mat& dst, cv::Mat& dst_mask); + cv::Mat dst_, dst_mask_; + cv::Rect dst_roi_; }; class FeatherBlender : public Blender { public: - FeatherBlender(float sharpness = 0.02f) : sharpness_(sharpness) {} + FeatherBlender(float sharpness = 0.02f) { setSharpness(sharpness); } + float sharpness() const { return sharpness_; } + void setSharpness(float val) { sharpness_ = val; } + + void prepare(cv::Rect dst_roi); + void feed(const cv::Mat &img, const cv::Mat &mask, cv::Point tl); + void blend(cv::Mat &dst, cv::Mat &dst_mask); private: - cv::Point blend(const std::vector &src, const std::vector &corners, const std::vector &masks, - cv::Mat &dst, cv::Mat &dst_mask); - float sharpness_; + cv::Mat weight_map_; + cv::Mat dst_weight_map_; }; class MultiBandBlender : public Blender { public: - MultiBandBlender(int num_bands = 7) : num_bands_(num_bands) {} - + MultiBandBlender(int num_bands = 7) { setNumBands(num_bands); } int numBands() const { return num_bands_; } void setNumBands(int val) { num_bands_ = val; } -private: - cv::Point blend(const std::vector &src, const std::vector &corners, const std::vector &masks, - cv::Mat& dst, cv::Mat& dst_mask); + void prepare(cv::Rect dst_roi); + void feed(const cv::Mat &img, const cv::Mat &mask, cv::Point tl); + void blend(cv::Mat &dst, cv::Mat &dst_mask); +private: int num_bands_; + std::vector dst_pyr_laplace_; + std::vector dst_band_weights_; }; ////////////////////////////////////////////////////////////////////////////// // Auxiliary functions -cv::Rect resultRoi(const std::vector &src, const std::vector &corners); - -cv::Point computeResultMask(const std::vector &masks, const std::vector &corners, cv::Mat &mask); - -cv::Point blendLinear(const std::vector &src, const std::vector &corners, const std::vector &weights, - cv::Mat& dst, cv::Mat& dst_weight); +cv::Rect resultRoi(const std::vector &corners, const std::vector &sizes); void normalize(const cv::Mat& weight, cv::Mat& src); diff --git a/modules/stitching/main.cpp b/modules/stitching/main.cpp index 933adc1bb..db773d405 100644 --- a/modules/stitching/main.cpp +++ b/modules/stitching/main.cpp @@ -71,7 +71,6 @@ int main(int argc, char* argv[]) } int64 t = getTickCount(); - LOGLN("Parsing params and reading images..."); for (int i = 1; i < argc; ++i) { if (string(argv[i]) == "--trygpu") @@ -189,7 +188,6 @@ int main(int argc, char* argv[]) else img_names.push_back(argv[i]); } - LOGLN("Parsing params and reading images, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); int num_images = static_cast(img_names.size()); if (num_images < 2) @@ -198,8 +196,8 @@ int main(int argc, char* argv[]) return -1; } + LOGLN("Reading images and finding features..."); t = getTickCount(); - LOGLN("Finding features..."); vector features(num_images); SurfFeaturesFinder finder(trygpu); Mat full_img, img; @@ -224,10 +222,10 @@ int main(int argc, char* argv[]) } finder(img, features[i]); } - LOGLN("Finding features, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); + LOGLN("Reading images and finding features, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); - t = getTickCount(); LOGLN("Pairwise matching... "); + t = getTickCount(); vector pairwise_matches; BestOf2NearestMatcher matcher(trygpu); if (user_match_conf) @@ -248,8 +246,8 @@ int main(int argc, char* argv[]) return -1; } - t = getTickCount(); LOGLN("Estimating rotations..."); + t = getTickCount(); HomographyBasedEstimator estimator; vector cameras; estimator(features, pairwise_matches, cameras); @@ -263,16 +261,16 @@ int main(int argc, char* argv[]) LOGLN("Initial focal length " << i << ": " << cameras[i].focal); } - t = getTickCount(); LOGLN("Bundle adjustment... "); + t = getTickCount(); BundleAdjuster adjuster(ba_space, conf_thresh); adjuster(features, pairwise_matches, cameras); LOGLN("Bundle adjustment, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); if (wave_correct) { - t = getTickCount(); LOGLN("Wave correcting..."); + t = getTickCount(); vector rmats; for (size_t i = 0; i < cameras.size(); ++i) rmats.push_back(cameras[i].R); @@ -292,9 +290,10 @@ int main(int argc, char* argv[]) nth_element(focals.begin(), focals.end(), focals.begin() + focals.size() / 2); float camera_focal = static_cast(focals[focals.size() / 2]); - t = getTickCount(); vector images(num_images); + LOGLN("Compose scaling..."); + t = getTickCount(); for (int i = 0; i < num_images; ++i) { Mat full_img = imread(img_names[i]); @@ -319,38 +318,54 @@ int main(int argc, char* argv[]) } vector corners(num_images); + vector sizes(num_images); vector masks_warped(num_images); vector images_warped(num_images); - t = getTickCount(); LOGLN("Warping images... "); + t = getTickCount(); + Ptr warper = Warper::createByCameraFocal(camera_focal, warp_type); for (int i = 0; i < num_images; ++i) { - corners[i] = (*warper)(images[i], static_cast(cameras[i].focal), cameras[i].R, images_warped[i]); - (*warper)(masks[i], static_cast(cameras[i].focal), cameras[i].R, masks_warped[i], INTER_NEAREST, BORDER_CONSTANT); + corners[i] = (*warper)(images[i], static_cast(cameras[i].focal), cameras[i].R, + images_warped[i]); + sizes[i] = images_warped[i].size(); + (*warper)(masks[i], static_cast(cameras[i].focal), cameras[i].R, masks_warped[i], + INTER_NEAREST, BORDER_CONSTANT); } + vector images_f(num_images); for (int i = 0; i < num_images; ++i) images_warped[i].convertTo(images_f[i], CV_32F); + LOGLN("Warping images, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); - t = getTickCount(); LOGLN("Finding seams..."); + t = getTickCount(); Ptr seam_finder = SeamFinder::createDefault(seam_find_type); (*seam_finder)(images_f, corners, masks_warped); LOGLN("Finding seams, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); - t = getTickCount(); LOGLN("Blending images..."); + t = getTickCount(); + Ptr blender = Blender::createDefault(blend_type); + if (blend_type == Blender::MULTI_BAND) + { // Ensure last pyramid layer area is about 1 pix - dynamic_cast((Blender*)(blender)) - ->setNumBands(static_cast(ceil(log(static_cast(images_f[0].size().area())) - / log(4.0)))); + MultiBandBlender* mb = dynamic_cast((Blender*)(blender)); + mb->setNumBands(static_cast(ceil(log(static_cast(images_f[0].size().area())) / log(4.0)))); + LOGLN("Multi-band blending num. bands: " << mb->numBands()); + } + + blender->prepare(corners, sizes); + for (int i = 0; i < num_images; ++i) + blender->feed(images_f[i], masks_warped[i], corners[i]); Mat result, result_mask; - (*blender)(images_f, corners, masks_warped, result, result_mask); + blender->blend(result, result_mask); + LOGLN("Blending images, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec"); imwrite(result_name, result);