Jacob的全景图像融合算法系列
OpenCV 尺度不变特征检测：SIFT、SURF、BRISK、ORB
OpenCV 匹配兴趣点：SIFT、SURF 和二值描述子
OpenCV 估算图像的投影关系：基础矩阵和RANSAC
OpenCV 单应矩阵应用：全景图像融合原理
图像融合：拉普拉斯金字塔融合算法
上一篇文章中讲到如何检测图像中的兴趣点，以便后续的局部图像分析。为了进行基于兴趣点的图像分析，我们需要构建多种表征方式，精确地描述每个关键点。这些描述子通常是二值类型、整数型或浮点数型组成的向量。好的描述子要具有足够的独特性和鲁棒性，能唯一地表示图像中的每个特征点，并且在亮度和视角变化时仍能提取出同一批点集。此外，尽量能够简洁，以减少计算资源的占用。
SIFT算法提出者 David Lowe
1.Harris、FAST
这两个特征检测算子不具有尺度不变等特性，所以使用这两个算子进行检测并匹配的效果一般不会很好。常见的方案是通过比较特征点附近的一个方块的像素集合的相似度，算法使用差的平方和（SSD），效果如下，这里不进行代码演示。可以看到，即使在视角差别不大的情况下，就已经有非常多的错误匹配项。
FAST & SSD
2.SIFT、SURF
so，尺度不变检测算子的优势就体现出来了。SIFT、SURF在检测出特征点之后，可以生成相应的描述子（Descriptor）。这些描述子具有的信息量比单纯地比较像素块的SSD多得多，于是能够更好地进行图像的匹配。至于描述子的数据结构，上一篇文章中提到过，这里不再赘述。其中SIFT是128维的向量，SURF则是检测Haar小波特征。
SIFT
SURF
直接进行匹配的话，也会有很多的错误匹配项，比上面那两位好不到哪里去。那么，算法研究员们想出了一些匹配策略，能够在一定程度上减少错误项。主要有：
- 交叉检查匹配项
交叉检查是指在第一幅图像匹配到第二幅图像后，再用第二幅图像的关键点再逐个跟第一幅的图像进行比较，只有在两个方向都匹配了同一个关键点时，才认为是一个有效的匹配项。
- 比率检测法
我们为每个关键点找到两个最佳的匹配项，方法是使用kNN最近邻（可以看我的这篇文章，其实在这里只是用了欧氏距离）。接下来计算排名第二的匹配项与排名第一的匹配项的差值之比（如果两个匹配项近乎相等，则结果接近为1）。比率过高的匹配项作为模糊匹配项，从结果中被排除。
- 匹配差值的阈值化
很简单，就是将差值过大的匹配项排除掉。
上面的一些匹配策略可以结合使用来提升匹配效果。代码实现如下
/******************************************************
* Created by 杨帮杰 on 10/5/18
* Right to use this code in any way you want without
* warranty, support or any guarantee of it working
* E-mail: yangbangjie1998@qq.com
* Association: SCAU 华南农业大学
******************************************************/
#include <iostream>
#include <vector>
#include <opencv2/core.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/features2d.hpp>
#include <opencv2/objdetect.hpp>
#include <opencv2/xfeatures2d.hpp>
#define IMAGE1_PATH "/home/jacob/下载/church01.jpg"
#define IMAGE2_PATH "/home/jacob/下载/church02.jpg"
#define IMAGE3_PATH "/home/jacob/下载/church03.jpg"
using namespace cv;
using namespace std;
int main()
{
/*******************SIFT、SURF：描述并匹配局部强度值模式***********************/
Mat image1= imread(IMAGE1_PATH,IMREAD_GRAYSCALE);
Mat image2= imread(IMAGE2_PATH,IMREAD_GRAYSCALE);
vector<KeyPoint> keypoints1;
vector<KeyPoint> keypoints2;
//创建SURF特征检测器
Ptr<Feature2D> ptrFeature2D = xfeatures2d::SURF::create(2000.0);
//创建SIFT特征检测器
//Ptr<Feature2D> ptrFeature2D = xfeatures2d::SIFT::create(74);
//检测特征点
ptrFeature2D->detect(image1,keypoints1);
ptrFeature2D->detect(image2,keypoints2);
Mat featureImage;
drawKeypoints(image1,keypoints1,featureImage,
Scalar(255,255,255),DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
imshow("SURF",featureImage);
cout << "Number of SURF keypoints (image 1): " << keypoints1.size() << endl;
cout << "Number of SURF keypoints (image 2): " << keypoints2.size() << endl;
//提取特征描述子
Mat descriptors1;
Mat descriptors2;
ptrFeature2D->compute(image1,keypoints1,descriptors1);
ptrFeature2D->compute(image2,keypoints2,descriptors2);
//使用L2范式（欧氏距离）进行配对
BFMatcher matcher(NORM_L2);
//进行交叉匹配
//BFMatcher matcher(NORM_L2, true);
vector<DMatch> matches;
matcher.match(descriptors1,descriptors2, matches);
Mat imageMatches;
drawMatches(
image1, keypoints1, // 1st image and its keypoints
image2, keypoints2, // 2nd image and its keypoints
matches, // the matches
imageMatches, // the image produced
Scalar(255, 255, 255), // color of lines
Scalar(255, 255, 255), // color of points
vector< char >(), // masks if any
DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS | DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
imshow("SURF Matches",imageMatches);
cout << "Number of matches: " << matches.size() << endl;
//使用比率检测法
//为每个关键点找出两个最佳匹配项
vector<vector<DMatch> > matches2;
matcher.knnMatch(descriptors1, descriptors2,
matches2,
2); // find the k (2) best matches
matches.clear();
//比率设定为0.6
double ratioMax= 0.6;
vector<vector<DMatch> >::iterator it;
for (it= matches2.begin(); it!= matches2.end(); ++it) {
// first best match/second best match
if ((*it)[0].distance/(*it)[1].distance < ratioMax) {
matches.push_back((*it)[0]);
}
}
drawMatches(
image1,keypoints1, // 1st image and its keypoints
image2,keypoints2, // 2nd image and its keypoints
matches, // the matches
imageMatches, // the image produced
Scalar(255,255,255), // color of lines
Scalar(255,255,255), // color of points
vector< char >(), // masks if any
DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS | DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
cout << "Number of matches (after ratio test): " << matches.size() << endl;
imshow("SURF Matches (ratio test at 0.6)",imageMatches);
//差值阈值化匹配，这里设为0.3
float maxDist = 0.3;
matches2.clear();
matcher.radiusMatch(descriptors1, descriptors2, matches2,
maxDist); // maximum acceptable distance
drawMatches(
image1, keypoints1, // 1st image and its keypoints
image2, keypoints2, // 2nd image and its keypoints
matches2, // the matches
imageMatches, // the image produced
Scalar(255, 255, 255), // color of lines
Scalar(255, 255, 255), // color of points
vector<vector< char >>(), // masks if any
DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS | DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
int nmatches = 0;
for (int i = 0; i< matches2.size(); i++)
nmatches += matches2[i].size();
cout << "Number of matches (with max radius): " << nmatches << endl;
imshow("SURF Matches (with max radius)", imageMatches);
/****************************尺度无关的匹配**************************************/
image1= imread(IMAGE1_PATH,CV_LOAD_IMAGE_GRAYSCALE);
image2= imread(IMAGE3_PATH,CV_LOAD_IMAGE_GRAYSCALE);
cout << "Number of SIFT keypoints (image 1): " << keypoints1.size() << endl;
cout << "Number of SIFT keypoints (image 2): " << keypoints2.size() << endl;
ptrFeature2D = xfeatures2d::SIFT::create();
ptrFeature2D->detectAndCompute(image1, noArray(), keypoints1, descriptors1);
ptrFeature2D->detectAndCompute(image2, noArray(), keypoints2, descriptors2);
matcher.match(descriptors1,descriptors2, matches);
//选取最好的50个
nth_element(matches.begin(),matches.begin()+50,matches.end());
matches.erase(matches.begin()+50,matches.end());
drawMatches(
image1, keypoints1, // 1st image and its keypoints
image2, keypoints2, // 2nd image and its keypoints
matches, // the matches
imageMatches, // the image produced
Scalar(255, 255, 255), // color of lines
Scalar(255, 255, 255), // color of points
vector<char>(), cv::DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS| cv::DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
imshow("Multi-scale SIFT Matches",imageMatches);
cout << "Number of matches: " << matches.size() << endl;
waitKey();
return 0;
}
结果如下
SURF和不同尺度下的SIFT匹配
SURF的比率检测和阈值化
3.ORB、BRISK
SIFT和SURF的描述子向量分别是浮点型的128位和64位，对他们操作耗资巨大，为了减少计算资源的使用，算法研究员们又引入了二值描述子的概念。其中ORB和BRISK生成的就是二值描述子。
其中，ORB实际上是在BRIEF描述子基础上构建的。实现过程是在关键点周围的邻域内随机选取一对像素点，从而创建一个二值描述子。比较这两个像素点的强度值，如果第一个点的强度值较大，就把对应描述子的位（bit）设为1，否则为0。对一批随机像素点进行上述处理，就产生了一个由若干位组成的描述子，通常在128到512位。对于ORB，为了解决旋转不变性，对256个随机点对进行旋转后进行判别。
ORB
二值描述子之间的比较一般使用Hamming Distance（汉明距离），表示的是两个等长子串或者二进制数之间不同位的个数，如
# 举例说明以下字符串间的汉明距离为：
"karolin" and "kathrin" is 3.
"karolin" and "kerstin" is 3.
1011101 and 1001001 is 2.
2173896 and 2233796 is 3.
代码实现如下
/******************************************************
* Created by 杨帮杰 on 10/5/18
* Right to use this code in any way you want without
* warranty, support or any guarantee of it working
* E-mail: yangbangjie1998@qq.com
* Association: SCAU 华南农业大学
******************************************************/
#include <iostream>
#include <vector>
#include <opencv2/core.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/features2d.hpp>
#include <opencv2/objdetect.hpp>
#include <opencv2/xfeatures2d.hpp>
#define IMAGE1_PATH "/home/jacob/下载/church01.jpg"
#define IMAGE2_PATH "/home/jacob/下载/church02.jpg"
#define IMAGE3_PATH "/home/jacob/下载/church03.jpg"
using namespace cv;
using namespace std;
int main()
{
Mat image1= imread(IMAGE1_PATH,CV_LOAD_IMAGE_GRAYSCALE);
Mat image2= imread(IMAGE2_PATH,CV_LOAD_IMAGE_GRAYSCALE);
vector<KeyPoint> keypoints1;
vector<KeyPoint> keypoints2;
Mat descriptors1;
Mat descriptors2;
//构建ORB特征检测器
//Ptr<Feature2D> feature =ORB::create(60);
//构建BRISK特征检测器
Ptr<Feature2D> feature = BRISK::create(80);
feature->detectAndCompute(image1, noArray(), keypoints1, descriptors1);
feature->detectAndCompute(image2, noArray(), keypoints2, descriptors2);
Mat featureImage;
drawKeypoints(image1,keypoints1,featureImage,
Scalar(255,255,255),DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
imshow("ORB",featureImage);
cout << "Number of ORB keypoints (image 1): " << keypoints1.size() << endl;
cout << "Number of ORB keypoints (image 2): " << keypoints2.size() << endl;
// 使用FREAK（快速视网膜关键点），配合BRISK
// feature = xfeatures2d::FREAK::create();
// feature->compute(image1, keypoints1, descriptors1);
// feature->compute(image1, keypoints2, descriptors2);
//二值描述子必须用Hamming规范
BFMatcher matcher(NORM_HAMMING);
vector<DMatch> matches;
matcher.match(descriptors1,descriptors2, matches);
Mat imageMatches;
drawMatches(
image1,keypoints1, // 1st image and its keypoints
image2,keypoints2, // 2nd image and its keypoints
matches, // the matches
imageMatches, // the image produced
Scalar(255,255,255), // color of lines
Scalar(255,255,255), // color of points
vector< char >(), // masks if any
DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS | DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
imshow("ORB Matches", imageMatches);
//imshow("BRISK Matches", imageMatches);
//imshow("FREAK with BRISK Matches", imageMatches);
cout << "Number of matches: " << matches.size() << endl;
waitKey();
return 0;
}
结果如下
ORB Matches
BRISK Matches
References:
OpenCV尺度不变特征检测：SIFT、SURF、BRISK、ORB
Hamming Distance (汉明距离)
【特征检测】ORB特征提取算法
opencv计算机视觉编程攻略（第三版） —— Robert
