OpenCV中显著性检测算法的使用

本文所用的 OpenCV 版本为 opencv-3.2.0，编程语言为 C++。

前言

　　OpenCV 中实现了两种显著性检测算法，分别为 Spectral Residual 算法,出自 Xiaodi Hou and Liqing Zhang. Saliency detection: A spectral residual approach. In Computer Vision and Pattern Recognition, 2007. CVPR'07. IEEE Conference on, pages 1–8. IEEE, 2007. 和 Fine Grained Saliency 算法,出自 Sebastian Montabone and Alvaro Soto. Human detection using a mobile platform and novel features derived from a visual saliency mechanism. In Image and Vision Computing, Vol. 28 Issue 3, pages 391–402. Elsevier, 2010.。这两种算法同样是在扩展包 opencv_contrib-3.2.0 中，也是由于 opencv 官方示例程序对初学者不友好（主要是 Shaun 境界不够 o(╯□╰)o），所以 Shaun 对照其官方文档重新整理了一下。

说明篇

　　使用 OpenCV 中实现的显著性检测算法进行显著性检测十分方便简洁，利用以下三个函数就可以：

创建 Spectral Residual 算法显著性检测对象：static Ptr<StaticSaliencySpectralResidual> cv::saliency::StaticSaliencySpectralResidual::create();

Spectral Residual 算法计算显著性图：bool cv::saliency::StaticSaliencySpectralResidual::computeSaliency(InputArray image, OutputArray saliencyMap);

Fine Grained Saliency 算法显著性检测对应的函数声明同 Spectral Residual 算法类似。

计算显著性图的二值图：bool cv::saliency::StaticSaliency::computeBinaryMap(InputArray _saliencyMap, OutputArray _binaryMap) ;

具体使用方法可参考实例篇。

实例篇

　　使用 OpenCV 中的显著性检测算法需要包含头文件#include <opencv2/saliency.hpp>，具体示例程序如下：

#include <opencv2/opencv.hpp>
#include <opencv2/saliency.hpp>

//******************************************************
// [opencv_contrib/modules/saliency/src/saliency.cpp](https://github.com/opencv/opencv_contrib/blob/b7dcf141507edbe544e75820c76769a7769223ac/modules/saliency/src/saliency.cpp)
//
//Ptr<Saliency> Saliency::create(const String& saliencyType)
//{
//	if (saliencyType == "SPECTRAL_RESIDUAL")
//		return makePtr<StaticSaliencySpectralResidual>();  //computeSaliency返回的是32FC1
//	else if (saliencyType == "FINE_GRAINED")
//		return makePtr<StaticSaliencyFineGrained>();	 //computeSaliency返回的是8UC1
//	else if (saliencyType == "BING")
//		return makePtr<ObjectnessBING>();
//	else if (saliencyType == "BinWangApr2014")
//		return makePtr<MotionSaliencyBinWangApr2014>();
//	return Ptr<Saliency>();
//}
//
// [opencv_contrib/modules/saliency/src/staticSaliency.cpp](https://github.com/opencv/opencv_contrib/blob/41b0a71ac826b1489d3e5c208ac7a95e58556caf/modules/saliency/src/staticSaliency.cpp)
//computeBinaryMap()要求输入的saliencyMap为浮点数（eg:32FC1）
//*****************************************************

void spectralResidualTest()
{
	cv::Mat src_img = cv::imread("../data/true.png", CV_LOAD_IMAGE_ANYDEPTH | CV_LOAD_IMAGE_ANYCOLOR);	// 载入最真实的原始图像
	cv::namedWindow("src_img", CV_WND_PROP_ASPECTRATIO);
	cv::imshow("src_img", src_img);


	// [OpenCV实现显著性检测中的谱残差法（Spectral Residual Method）涉及到了傅立叶正反变换](http://blog.csdn.net/kena_m/article/details/49406687)
	if (src_img.empty())
		exit(-1);
	if (src_img.channels() == 3)
		cv::cvtColor(src_img, src_img, CV_BGR2GRAY);
	cv::Mat planes[] = { cv::Mat_<float>(src_img), cv::Mat::zeros(src_img.size(), CV_32F) };
	cv::Mat complex_img; //复数矩阵
	merge(planes, 2, complex_img); //把单通道矩阵组合成复数形式的双通道矩阵
	dft(complex_img, complex_img);  // 使用离散傅立叶变换

	//对复数矩阵进行处理，方法为谱残差
	cv::Mat magnitude, phase_angle, mag_mean;
	cv::Mat real_part, imaginary_part;
	split(complex_img, planes); //分离复数到实部和虚部
	real_part = planes[0]; //实部
	imaginary_part = planes[1]; //虚部
	cv::magnitude(real_part, imaginary_part, magnitude); //计算幅值
	phase(real_part, imaginary_part, phase_angle); //计算相角

	float *pre, *pim, *pm, *pp;
	//对幅值进行对数化
	for (int i = 0; i < magnitude.rows; i++)
	{
		pm = magnitude.ptr<float>(i);
		for (int j = 0; j < magnitude.cols; j++)
		{
			*pm = log(*pm);
			pm++;
		}
	}
	blur(magnitude, mag_mean, cv::Size(5, 5)); //对数谱的均值滤波
	magnitude = magnitude - mag_mean; //求取对数频谱残差
	//把对数谱残差的幅值和相角划归到复数形式
	for (int i = 0; i < magnitude.rows; i++)
	{
		pre = real_part.ptr<float>(i);
		pim = imaginary_part.ptr<float>(i);
		pm = magnitude.ptr<float>(i);
		pp = phase_angle.ptr<float>(i);
		for (int j = 0; j < magnitude.cols; j++)
		{
			*pm = exp(*pm);
			*pre = *pm * cos(*pp);
			*pim = *pm * sin(*pp);
			pre++;
			pim++;
			pm++;
			pp++;
		}
	}
	cv::Mat planes1[] = { cv::Mat_<float>(real_part), cv::Mat_<float>(imaginary_part) };

	merge(planes1, 2, complex_img); //重新整合实部和虚部组成双通道形式的复数矩阵
	idft(complex_img, complex_img, cv::DFT_SCALE); // 傅立叶反变换
	split(complex_img, planes); //分离复数到实部和虚部
	real_part = planes[0];
	imaginary_part = planes[1];
	cv::magnitude(real_part, imaginary_part, magnitude); //计算幅值和相角
	for (int i = 0; i < magnitude.rows; i++)
	{
		pm = magnitude.ptr<float>(i);
		for (int j = 0; j < magnitude.cols; j++)
		{
			*pm = (*pm) * (*pm);
			pm++;
		}
	}
	GaussianBlur(magnitude, magnitude, cv::Size(7, 7), 2.5, 2.5);
	cv::Mat invDFT, invDFTcvt;
	normalize(magnitude, invDFT, 0, 255, cv::NORM_MINMAX); //归一化到[0,255]供显示
	invDFT.convertTo(invDFTcvt, CV_8U); //转化成CV_8U型
	cv::namedWindow("SpectualResidual", CV_WND_PROP_ASPECTRATIO);
	cv::imshow("SpectualResidual", invDFTcvt);

	cv::Mat thresholded;
	cv::threshold(invDFTcvt, thresholded, 0, 255, CV_THRESH_OTSU);
	cv::namedWindow("Thresholded Image", CV_WND_PROP_ASPECTRATIO);
	cv::imshow("Thresholded Image", thresholded);

	cv::Mat eroded;
	// 纵向腐蚀
	cv::erode(thresholded, eroded, cv::Mat(5, 1, CV_8UC1, cv::Scalar(1)), cv::Point(-1, -1), 3);	// cv::Point(-1,-1)为默认参数，代表原点（描点）为矩阵中心
	cv::namedWindow("eroded Image", CV_WND_PROP_ASPECTRATIO);
	cv::imshow("eroded Image", eroded);

	//cv::Mat thresholded;
	cv::threshold(eroded, thresholded, 60, 255, CV_THRESH_BINARY);
	cv::namedWindow("Thresholded eroded Image", CV_WND_PROP_ASPECTRATIO);
	cv::imshow("Thresholded eroded Image", thresholded);
}

 // 显著性检测算法基类
void saliencyTest()
{
	cv::Mat src_img = cv::imread("../data/true.png", CV_LOAD_IMAGE_ANYDEPTH | CV_LOAD_IMAGE_ANYCOLOR);
	cv::namedWindow("src_img", CV_WND_PROP_ASPECTRATIO);
	cv::imshow("src_img", src_img);

	if (src_img.empty())
		exit(-1);
	if (src_img.channels() == 3)
		cv::cvtColor(src_img, src_img, CV_BGR2GRAY);

	cv::Ptr<cv::saliency::Saliency> saliency_algorithm = cv::saliency::Saliency::create("SPECTRAL_RESIDUAL");	// FINE_GRAINED为Fine Grained Saliency算法
	cv::Mat saliency_map;
	if (saliency_algorithm->computeSaliency(src_img, saliency_map))	// 计算显著性图
	{
		cv::namedWindow("SR saliency map", CV_WND_PROP_ASPECTRATIO);
		cv::imshow("SR saliency map", saliency_map);

		cv::Mat saliency_map_show(saliency_map.size(), CV_8UC1);
		normalize(saliency_map, saliency_map_show, 0, 255, CV_MINMAX); //归一化到[0,255]供显示
		saliency_map_show.convertTo(saliency_map_show, CV_8U); //转化成CV_8U型
		cv::namedWindow("saliency_map_show", CV_WND_PROP_ASPECTRATIO);
		cv::imshow("saliency_map_show", saliency_map_show);

		cv::Mat binary_map;
		cv::saliency::StaticSaliencySpectralResidual spec;
		if (spec.computeBinaryMap(saliency_map, binary_map))	// 对显著性图进行二值化
		{
			cv::namedWindow("binary map", CV_WND_PROP_ASPECTRATIO);
			cv::imshow("binary map", binary_map);
		}
	}
}

// Fine Grained Saliency算法
void FGSTest()
{
	cv::Mat src_img = cv::imread("../data/true.png", CV_LOAD_IMAGE_ANYDEPTH | CV_LOAD_IMAGE_ANYCOLOR);
	cv::namedWindow("src_img", CV_WND_PROP_ASPECTRATIO);
	cv::imshow("src_img", src_img);

	if (src_img.empty())
		exit(-1);
	if (src_img.channels() == 3)
		cv::cvtColor(src_img, src_img, CV_BGR2GRAY);

	cv::Ptr<cv::saliency::StaticSaliencyFineGrained> fgs = cv::saliency::StaticSaliencyFineGrained::create();
	cv::Mat fgs_saliency_map;
	fgs->computeSaliency(src_img, fgs_saliency_map);
	cv::namedWindow("FGS saliency map", CV_WND_PROP_ASPECTRATIO);
	cv::imshow("FGS saliency map", fgs_saliency_map);
	//cv::imwrite("../data/T_S.png", fgs_saliency_map);

	cv::Mat binary_map;
	cv::threshold(fgs_saliency_map, binary_map, 0, 255, CV_THRESH_OTSU);
	cv::namedWindow("binary map", CV_WND_PROP_ASPECTRATIO);
	cv::imshow("binary map", binary_map);
	//cv::imwrite("../data/T_S_B.png", binary_map);
}

// Spectral Residual算法
void SRTest()
{
	cv::Mat src_img = cv::imread("../data/true.png", CV_LOAD_IMAGE_ANYDEPTH | CV_LOAD_IMAGE_ANYCOLOR);
	cv::namedWindow("src_img", CV_WND_PROP_ASPECTRATIO);
	cv::imshow("src_img", src_img);

	if (src_img.empty())
		exit(-1);
	if (src_img.channels() == 3)
		cv::cvtColor(src_img, src_img, CV_BGR2GRAY);

	cv::Ptr<cv::saliency::StaticSaliencySpectralResidual> sr = cv::saliency::StaticSaliencySpectralResidual::create();
	cv::Mat sr_saliency_map;
	sr->computeSaliency(src_img, sr_saliency_map);
	cv::namedWindow("SR saliency map", CV_WND_PROP_ASPECTRATIO);
	cv::imshow("SR saliency map", sr_saliency_map);

	cv::Mat binary_map;
	sr->computeBinaryMap(sr_saliency_map, binary_map);
	cv::namedWindow("binary map", CV_WND_PROP_ASPECTRATIO);
	cv::imshow("binary map", binary_map);
}

int main(int argc, char *argv[])
{
	//spectralResidualTest();
	//saliencyTest();
	//FGSTest();
	SRTest();

	while (cv::waitKey(0) != 27) { }

	return 0;
}

以上代码在 Win10 VS2013 中编译运行成功。

　　这里面有个小东西需要注意，就是 computeBinaryMap() 函数，看其文档描述其中使用 K-means 算法和 Otsu 算法对显著性图进行二值化处理，其输入的显著性图数据类型应该为浮点数，OpenCV 中 Spectral Residual 算法 computeSaliency() 返回的结果为浮点数，而 Fine Grained Saliency 算法 computeSaliency() 返回的结果却是整型数据，所以这一点需要注意 Fine Grained Saliency 算法返回的结果不能直接使用 computeBinaryMap() 函数，一般对其结果直接使用 OTSU 算法进行阈值分割即可。

后记

　　本文使用的这两种算法在 Shaun 的电脑上运行时间都较长，基本不可能用来处理视频流，而且在 Shaun 的这次实验中效果也不太理想，毕竟这是用来处理静态图像的两种显著性方法。不过 OpenCV 中也有用来处理视频流的显著性检测算法，其为 BING 算法,出自Ming-Ming Cheng, Ziming Zhang, Wen-Yan Lin, and Philip Torr. Bing: Binarized normed gradients for objectness estimation at 300fps. In IEEE CVPR, 2014.，实际上这是一种快速提取目标候选框的算法。