如题，基于OpenCV3.4.13+VS2015做了个双摄像头实时拼接的代码，是一个大项目的一个baseline的一部分。下面先说配环境再给代码。
关于OpenCV+VS的环境配置网上已经有很多了，因为这份代码用到了OpenCV_Contrib里面的一些东西，所以这里赘述一下，更详细的过程可以参考这篇博客。
使用OpenCV_Contrib就得编译，编译就很麻烦，比配置还麻烦，因此我做了个资源集合，有需要的兄弟可以自取。
链接（2020.12.15更新）：https://pan.baidu.com/s/14_EKEYvEQVmS5Bls8lcnxw
提取码：bj29
假设你把我这个名为opencv的资源集合放在了$dir$下，$dir$的具体位置根据你的选择而定。则配置过程为：
此电脑->右键属性->高级系统设置->环境变量->系统变量->Path->新建，写入：
$dir$\opencv\build\x64\vc14\bin
然后一路确定。
VS2015，菜单栏->视图->其他窗口->属性管理器，对Debug|X64进行配置，右键Microsoft.Cpp.x64.user，点击属性，然后
VC++->包含目录，加入：
$dir$\opencv\build\include
$dir$\opencv\build\include\opencv
$dir$\opencv\build\include\opencv2
VC++->库目录，加入：
$dir$\opencv\build\x64\vc14\lib
注意这里的vc14是跟我的vs2015对应的，如果你不是vs2015就别选vc14目录，找你的vs对应的vc目录。
连接器->输入->附加依赖项，加入：
opencv_aruco341d.lib
opencv_bgsegm341d.lib
opencv_bioinspired341d.lib
opencv_calib3d341d.lib
opencv_ccalib341d.lib
opencv_core341d.lib
opencv_datasets341d.lib
opencv_dnn341d.lib
opencv_dnn_objdetect341d.lib
opencv_dpm341d.lib
opencv_face341d.lib
opencv_features2d341d.lib
opencv_flann341d.lib
opencv_fuzzy341d.lib
opencv_hdf341d.lib
opencv_hfs341d.lib
opencv_highgui341d.lib
opencv_imgcodecs341d.lib
opencv_imgproc341d.lib
opencv_img_hash341d.lib
opencv_line_descriptor341d.lib
opencv_ml341d.lib
opencv_objdetect341d.lib
opencv_optflow341d.lib
opencv_phase_unwrapping341d.lib
opencv_photo341d.lib
opencv_plot341d.lib
opencv_reg341d.lib
opencv_rgbd341d.lib
opencv_saliency341d.lib
opencv_shape341d.lib
opencv_stereo341d.lib
opencv_stitching341d.lib
opencv_structured_light341d.lib
opencv_superres341d.lib
opencv_surface_matching341d.lib
opencv_text341d.lib
opencv_tracking341d.lib
opencv_video341d.lib
opencv_videoio341d.lib
opencv_videostab341d.lib
opencv_viz341d.lib
opencv_xfeatures2d341d.lib
opencv_ximgproc341d.lib
opencv_xobjdetect341d.lib
opencv_xphoto341d.lib
这里一共有46个.lib文件，有的是OpenCV3.4.13带的，有的是我用cmake和opencv_contrib编译出来的，而且都是对应于3.4.13版本的，如果你不是3.4.13版本你就写你对应的版本和.lib名称。而且我当时没有编译world集合lib，所以这里有46个。
调试->选项->常规 勾选启动源服务器支持。
注意你调试的时候也要选Debug，x64.
将上述46个名称对应的 .dll 文件全都放到C:\Windows\System32下。注意是.dll文件，位置在：
$dir$\opencv\build\x64\vc14\bin
同样的，vc14是对应我的vs2015的，如果你不是vs2015就别选这个目录。
这里给出一个简单用Stitcher类做拼接的代码测试一下opencv+contrib的环境：
#include <stdio.h>
#include "myhead.h"
#include <opencv2\opencv.hpp>
#include <opencv2\stitching.hpp>
using namespace std;
using namespace cv;
int main()
{
vector< Mat > vImg;
Mat rImg;
vImg.push_back(imread("10.jpg"));
vImg.push_back(imread("11.jpg"));
Stitcher stitcher = Stitcher::createDefault();
unsigned long AAtime = 0, BBtime = 0; //check processing time
AAtime = getTickCount(); //check processing time
Stitcher::Status status = stitcher.stitch(vImg, rImg);
BBtime = getTickCount(); //check processing time
printf("Time consuming: %.2lf sec \n", (BBtime - AAtime) / getTickFrequency()); //check processing time
if (Stitcher::OK == status)
imshow("Stitching Result", rImg);
else
printf("Stitching fail.");
imwrite("test.jpg", rImg);
waitKey(0);
}
10.jpg和11.jpg分别如下
如果得到以下显示，那么恭喜你，环境配好了。
直接上代码。
#include <iostream>
#include <fstream>
#include <string>
#include <time.h>
#include "opencv2/opencv_modules.hpp"
#include <opencv2/core/utility.hpp>
#include "opencv2/imgcodecs.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/stitching/detail/autocalib.hpp"
#include "opencv2/stitching/detail/blenders.hpp"
#include "opencv2/stitching/detail/timelapsers.hpp"
#include "opencv2/stitching/detail/camera.hpp"
#include "opencv2/stitching/detail/exposure_compensate.hpp"
#include "opencv2/stitching/detail/matchers.hpp"
#include "opencv2/stitching/detail/motion_estimators.hpp"
#include "opencv2/stitching/detail/seam_finders.hpp"
#include "opencv2/stitching/detail/warpers.hpp"
#include "opencv2/stitching/warpers.hpp"
#define ENABLE_LOG 1
#define LOG(msg) std::cout << msg
#define LOGLN(msg) std::cout << msg << std::endl
using namespace std;
using namespace cv;
using namespace cv::detail;
// Default command line args
vector<int> img_names;
bool preview = false;
bool try_cuda = false;
double work_megapix = 0.6;
double seam_megapix = 0.1;
double compose_megapix = -1;
float conf_thresh = 1.f;
string features_type = "surf";
string matcher_type = "homography";
string estimator_type = "homography";
string ba_cost_func = "ray";
string ba_refine_mask = "xxxxx";
bool do_wave_correct = true;
WaveCorrectKind wave_correct = detail::WAVE_CORRECT_HORIZ;
bool save_graph = false;
std::string save_graph_to;
string warp_type = "spherical"; //plane spherical
int expos_comp_type = ExposureCompensator::GAIN_BLOCKS;
float match_conf = 0.3f;
string seam_find_type = "gc_color";
int blend_type = Blender::MULTI_BAND;
int timelapse_type = Timelapser::AS_IS;
float blend_strength = 5;
string result_name = "result.jpg";
bool timelapse = false;
int range_width = -1;
int main(int argc, char* argv[])
{
#if ENABLE_LOG
int64 app_start_time = getTickCount();
#endif
#if 0
cv::setBreakOnError(true);
#endif
VideoCapture cap0(0), cap1(1);
Mat framecap[10];
if (cap0.isOpened() && cap1.isOpened())
{
cout << "*** ***" << endl;
cout << "摄像头已启动！" << endl;
}
cap0.set(CV_CAP_PROP_FOCUS, 0);
cap1.set(CV_CAP_PROP_FOCUS, 0);
int k = 50;
while (k--)
{
if (cap0.read(framecap[0]) && cap1.read(framecap[1]))
{
imwrite("frame0.bmp", framecap[0]);
imwrite("frame1.bmp", framecap[1]);
}
}
img_names.push_back(0);
img_names.push_back(1);
// Check if have enough images
int num_images = static_cast<int>(img_names.size());
if (num_images < 2)
{
LOGLN("Need more images 1");
getchar();
return -1;
}
double work_scale = 1, seam_scale = 1, compose_scale = 1;
bool is_work_scale_set = false, is_seam_scale_set = false, is_compose_scale_set = false;
LOGLN("Finding features...");
#if ENABLE_LOG
int64 t = getTickCount();
#endif
Ptr<FeaturesFinder> finder;
if (features_type == "surf")
{
#ifdef HAVE_OPENCV_XFEATURES2D
if (try_cuda && cuda::getCudaEnabledDeviceCount() > 0)
finder = makePtr<SurfFeaturesFinderGpu>();
else
#endif
finder = makePtr<SurfFeaturesFinder>();
}
else
{
cout << "Unknown 2D features type: '" << features_type << "'.\n";
getchar();
return -1;
}
Mat full_img, img;
vector<ImageFeatures> features(num_images);
vector<Mat> images(num_images);
vector<Size> full_img_sizes(num_images);
double seam_work_aspect = 1;
for (int i = 0; i < num_images; ++i)
{
full_img = framecap[i];
full_img_sizes[i] = full_img.size();
if (full_img.empty())
{
LOGLN("Can't open image " << img_names[i]);
getchar();
return -1;
}
if (work_megapix < 0)
{
img = full_img;
work_scale = 1;
is_work_scale_set = true;
}
else
{
if (!is_work_scale_set)
{
work_scale = min(1.0, sqrt(work_megapix * 1e6 / full_img.size().area()));
is_work_scale_set = true;
}
resize(full_img, img, Size(), work_scale, work_scale, INTER_LINEAR_EXACT);
}
if (!is_seam_scale_set)
{
seam_scale = min(1.0, sqrt(seam_megapix * 1e6 / full_img.size().area()));
seam_work_aspect = seam_scale / work_scale;
is_seam_scale_set = true;
}
(*finder)(img, features[i]);
features[i].img_idx = i;
LOGLN("Features in image #" << i + 1 << ": " << features[i].keypoints.size());
resize(full_img, img, Size(), seam_scale, seam_scale, INTER_LINEAR_EXACT);
images[i] = img.clone();
}
finder->collectGarbage();
full_img.release();
img.release();
LOGLN("Finding features, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec");
LOG("Pairwise matching");
#if ENABLE_LOG
t = getTickCount();
#endif
vector<MatchesInfo> pairwise_matches;
Ptr<FeaturesMatcher> matcher;
if (matcher_type == "affine")
matcher = makePtr<AffineBestOf2NearestMatcher>(false, try_cuda, match_conf);
else if (range_width == -1)
matcher = makePtr<BestOf2NearestMatcher>(try_cuda, match_conf);
else
matcher = makePtr<BestOf2NearestRangeMatcher>(range_width, try_cuda, match_conf);
(*matcher)(features, pairwise_matches);
matcher->collectGarbage();
LOGLN("Pairwise matching, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec");
// Leave only images we are sure are from the same panorama
vector<int> indices = leaveBiggestComponent(features, pairwise_matches, conf_thresh);
vector<Mat> img_subset;
vector<int> img_names_subset;
vector<Size> full_img_sizes_subset;
for (size_t i = 0; i < indices.size(); ++i)
{
img_names_subset.push_back(img_names[indices[i]]);
img_subset.push_back(images[indices[i]]);
full_img_sizes_subset.push_back(full_img_sizes[indices[i]]);
}
images = img_subset;
img_names = img_names_subset;
full_img_sizes = full_img_sizes_subset;
// Check if we still have enough images
num_images = static_cast<int>(img_names.size());
if (num_images < 2)
{
LOGLN("Need more images 2");
getchar();
return -1;
}
Ptr<Estimator> estimator = makePtr<HomographyBasedEstimator>();
vector<CameraParams> cameras;
if (!(*estimator)(features, pairwise_matches, cameras))
{
cout << "Homography estimation failed.\n";
getchar();
return -1;
}
for (size_t i = 0; i < cameras.size(); ++i)
{
Mat R;
cameras[i].R.convertTo(R, CV_32F);
cameras[i].R = R;
LOGLN("Initial camera intrinsics #" << indices[i] + 1 << ":\nK:\n" << cameras[i].K() << "\nR:\n" << cameras[i].R);
}
Ptr<detail::BundleAdjusterBase> adjuster = makePtr<detail::BundleAdjusterRay>();
adjuster->setConfThresh(conf_thresh);
Mat_<uchar> refine_mask = Mat::zeros(3, 3, CV_8U);
if (ba_refine_mask[0] == 'x') refine_mask(0, 0) = 1;
if (ba_refine_mask[1] == 'x') refine_mask(0, 1) = 1;
if (ba_refine_mask[2] == 'x') refine_mask(0, 2) = 1;
if (ba_refine_mask[3] == 'x') refine_mask(1, 1) = 1;
if (ba_refine_mask[4] == 'x') refine_mask(1, 2) = 1;
adjuster->setRefinementMask(refine_mask);
if (!(*adjuster)(features, pairwise_matches, cameras))
{
cout << "Camera parameters adjusting failed.\n";
getchar();
return -1;
}
// Find median focal length
vector<double> focals;
for (size_t i = 0; i < cameras.size(); ++i)
{
LOGLN("Camera #" << indices[i] + 1 << ":\nK:\n" << cameras[i].K() << "\nR:\n" << cameras[i].R);
focals.push_back(cameras[i].focal);
}
sort(focals.begin(), focals.end());
float warped_image_scale;
if (focals.size() % 2 == 1)
warped_image_scale = static_cast<float>(focals[focals.size() / 2]);
else
warped_image_scale = static_cast<float>(focals[focals.size() / 2 - 1] + focals[focals.size() / 2]) * 0.5f;
if (do_wave_correct)
{
vector<Mat> rmats;
for (size_t i = 0; i < cameras.size(); ++i)
rmats.push_back(cameras[i].R.clone());
waveCorrect(rmats, wave_correct);
for (size_t i = 0; i < cameras.size(); ++i)
cameras[i].R = rmats[i];
}
LOGLN("Warping images (auxiliary)... ");
#if ENABLE_LOG
t = getTickCount();
#endif
vector<Point> corners(num_images);
vector<UMat> masks_warped(num_images);
vector<UMat> images_warped(num_images);
vector<Size> sizes(num_images);
vector<UMat> masks(num_images);
// Preapre images masks
for (int i = 0; i < num_images; ++i)
{
masks[i].create(images[i].size(), CV_8U);
masks[i].setTo(Scalar::all(255));
}
// Warp images and their masks
Ptr<WarperCreator> warper_creator;
#ifdef HAVE_OPENCV_CUDAWARPING
if (try_cuda && cuda::getCudaEnabledDeviceCount() > 0)
{
warper_creator = makePtr<cv::PlaneWarperGpu>();
}
else
#endif
{
warper_creator = makePtr<cv::PlaneWarper>();
}
if (!warper_creator)
{
cout << "Can't create the following warper '" << warp_type << "'\n";
return 1;
}
Ptr<RotationWarper> warper = warper_creator->create(static_cast<float>(warped_image_scale * seam_work_aspect));
for (int i = 0; i < num_images; ++i)
{
Mat_<float> K;
cameras[i].K().convertTo(K, CV_32F);
float swa = (float)seam_work_aspect;
K(0, 0) *= swa; K(0, 2) *= swa;
K(1, 1) *= swa; K(1, 2) *= swa;
corners[i] = warper->warp(images[i], K, cameras[i].R, INTER_LINEAR, BORDER_REFLECT, images_warped[i]);
sizes[i] = images_warped[i].size();
warper->warp(masks[i], K, cameras[i].R, INTER_NEAREST, BORDER_CONSTANT, masks_warped[i]);
}
vector<UMat> images_warped_f(num_images);
for (int i = 0; i < num_images; ++i)
images_warped[i].convertTo(images_warped_f[i], CV_32F);
LOGLN("Warping images, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec");
Ptr<ExposureCompensator> compensator = ExposureCompensator::createDefault(expos_comp_type);
compensator->feed(corners, images_warped, masks_warped);
Ptr<SeamFinder> seam_finder;
if (seam_find_type == "gc_color")
{
#ifdef HAVE_OPENCV_CUDALEGACY
if (try_cuda && cuda::getCudaEnabledDeviceCount() > 0)
seam_finder = makePtr<detail::GraphCutSeamFinderGpu>(GraphCutSeamFinderBase::COST_COLOR);
else
#endif
seam_finder = makePtr<detail::GraphCutSeamFinder>(GraphCutSeamFinderBase::COST_COLOR);
}
if (!seam_finder)
{
cout << "Can't create the following seam finder '" << seam_find_type << "'\n";
return 1;
}
seam_finder->find(images_warped_f, corners, masks_warped);
// Release unused memory
images.clear();
images_warped.clear();
images_warped_f.clear();
masks.clear();
LOGLN("Compositing...");
#if ENABLE_LOG
t = getTickCount();
#endif
Mat img_warped, img_warped_s;
Mat dilated_mask, seam_mask, mask, mask_warped;
Ptr<Blender> blender;
Ptr<Timelapser> timelapser;
//double compose_seam_aspect = 1;
double compose_work_aspect = 1;
for (int img_idx = 0; img_idx < num_images; ++img_idx)
{
LOGLN("Compositing image #" << indices[img_idx] + 1);
// Read image and resize it if necessary
full_img = framecap[img_names[img_idx]];
if (!is_compose_scale_set)
{
if (compose_megapix > 0)
compose_scale = min(1.0, sqrt(compose_megapix * 1e6 / full_img.size().area()));
is_compose_scale_set = true;
// Compute relative scales
//compose_seam_aspect = compose_scale / seam_scale;
compose_work_aspect = compose_scale / work_scale;
// Update warped image scale
warped_image_scale *= static_cast<float>(compose_work_aspect);
warper = warper_creator->create(warped_image_scale);
// Update corners and sizes
for (int i = 0; i < num_images; ++i)
{
// Update intrinsics
cameras[i].focal *= compose_work_aspect;
cameras[i].ppx *= compose_work_aspect;
cameras[i].ppy *= compose_work_aspect;
// Update corner and size
Size sz = full_img_sizes[i];
if (std::abs(compose_scale - 1) > 1e-1)
{
sz.width = cvRound(full_img_sizes[i].width * compose_scale);
sz.height = cvRound(full_img_sizes[i].height * compose_scale);
}
Mat K;
cameras[i].K().convertTo(K, CV_32F);
Rect roi = warper->warpRoi(sz, K, cameras[i].R);
corners[i] = roi.tl();
sizes[i] = roi.size();
}
}
if (abs(compose_scale - 1) > 1e-1)
resize(full_img, img, Size(), compose_scale, compose_scale, INTER_LINEAR_EXACT);
else
img = full_img;
full_img.release();
Size img_size = img.size();
Mat K;
cameras[img_idx].K().convertTo(K, CV_32F);
// Warp the current image
warper->warp(img, K, cameras[img_idx].R, INTER_LINEAR, BORDER_REFLECT, img_warped);
// Warp the current image mask
mask.create(img_size, CV_8U);
mask.setTo(Scalar::all(255));
warper->warp(mask, K, cameras[img_idx].R, INTER_NEAREST, BORDER_CONSTANT, mask_warped);
// Compensate exposure
compensator->apply(img_idx, corners[img_idx], img_warped, mask_warped);
img_warped.convertTo(img_warped_s, CV_16S);
img_warped.release();
img.release();
mask.release();
dilate(masks_warped[img_idx], dilated_mask, Mat());
resize(dilated_mask, seam_mask, mask_warped.size(), 0, 0, INTER_LINEAR_EXACT);
mask_warped = seam_mask & mask_warped;
if (!blender && !timelapse)
{
blender = Blender::createDefault(blend_type, try_cuda);
Size dst_sz = resultRoi(corners, sizes).size();
float blend_width = sqrt(static_cast<float>(dst_sz.area())) * blend_strength / 100.f;
if (blend_width < 1.f)
blender = Blender::createDefault(Blender::NO, try_cuda);
else if (blend_type == Blender::MULTI_BAND)
{
MultiBandBlender* mb = dynamic_cast<MultiBandBlender*>(blender.get());
mb->setNumBands(static_cast<int>(ceil(log(blend_width) / log(2.)) - 1.));
LOGLN("Multi-band blender, number of bands: " << mb->numBands());
}
blender->prepare(corners, sizes);
}
else if (!timelapser && timelapse)
{
timelapser = Timelapser::createDefault(timelapse_type);
timelapser->initialize(corners, sizes);
}
blender->feed(img_warped_s, mask_warped, corners[img_idx]);
}
if (!timelapse)
{
Mat result, result_mask;
blender->blend(result, result_mask);
LOGLN("Compositing, time: " << ((getTickCount() - t) / getTickFrequency()) << " sec");
}
LOGLN("Finished preparing, total time: " << ((getTickCount() - app_start_time) / getTickFrequency()) << " sec");
images.clear();
images_warped.clear();
images_warped_f.clear();
masks.clear();
int numberofframe = 0;
time_t beforerun = time(0);
while (cap0.read(framecap[0]) && cap1.read(framecap[1]))
{
//imshow("cap0", framecap[0]);
//imshow("cap1", framecap[1]);
//Mat img_warped, img_warped_s;
//Mat dilated_mask, seam_mask, mask, mask_warped;
Ptr<Blender> blender;
for (int img_idx = 0; img_idx < num_images; ++img_idx)
{
full_img = framecap[img_idx];
if (abs(compose_scale - 1) > 1e-1)
resize(full_img, img, Size(), compose_scale, compose_scale, INTER_LINEAR_EXACT);
else
img = full_img;
full_img.release();
Size img_size = img.size();
Mat K;
cameras[img_idx].K().convertTo(K, CV_32F);
// Warp the current image
warper->warp(img, K, cameras[img_idx].R, INTER_LINEAR, BORDER_REFLECT, img_warped);
// Warp the current image mask
mask.create(img_size, CV_8U);
mask.setTo(Scalar::all(255));
warper->warp(mask, K, cameras[img_idx].R, INTER_NEAREST, BORDER_CONSTANT, mask_warped);
// Compensate exposure
compensator->apply(img_idx, corners[img_idx], img_warped, mask_warped);
img_warped.convertTo(img_warped_s, CV_16S);
img_warped.release();
img.release();
mask.release();
dilate(masks_warped[img_idx], dilated_mask, Mat());
resize(dilated_mask, seam_mask, mask_warped.size(), 0, 0, INTER_LINEAR_EXACT);
mask_warped = seam_mask & mask_warped;
if (!blender && !timelapse)
{
blender = Blender::createDefault(blend_type, try_cuda);
Size dst_sz = resultRoi(corners, sizes).size();
float blend_width = sqrt(static_cast<float>(dst_sz.area())) * blend_strength / 100.f;
if (blend_width < 1.f)
blender = Blender::createDefault(Blender::NO, try_cuda);
blender->prepare(corners, sizes);
}
blender->feed(img_warped_s, mask_warped, corners[img_idx]);
}
Mat result, result_mask;
blender->blend(result, result_mask);
Mat frame;
result.convertTo(frame, CV_8UC1);
imshow("stitch", frame);
numberofframe++;
if (cvWaitKey(10) == 27) break;
}
time_t afterrun = time(0);
cout << "帧率约为" << numberofframe / (afterrun - beforerun) << "帧/s" << endl;
getchar();
}
这份代码是从opencv examples里面的stitching_detailed.cpp改过来的，在CPU的基础上基本做到极限了，可优化的空间我感觉是不大。基于GPU加速还可以更快一点，这是后续工作了。也可以很容易地改为3摄像头或者更多摄像头的实时拼接。
代码整体应该没有大的bug，如果提示Need more images 2并且卡住的话表面双摄像头采集到的结果无法拼接起来，你需要调整一下两个摄像头的位置重新拼接试试。
如有问题可以随时与我联系！