v0.1.5 支持grab图片上下镜像检测; v0.1.4 支持单通道图片

ReadMe.txt

@@ -1,4 +1,6 @@
 v0.1.0 初始版本，实现整体流程，模型待优化。
 v0.1.1 分开抓取和切割的接口。
 v0.1.2 增加棋盘格角点识别接口。
-v0.1.3 增加图片切分功能
+v0.1.3 增加图片切分功能
+v0.1.4 支持单通道图片
+v0.1.5 支持grab图片上下镜像检测

config.cpp

@@ -31,6 +31,7 @@ namespace graft_cv{
 			<< "grid_row_grab" << m_cparam->grid_row_grab
 			<< "grid_col_grab" << m_cparam->grid_col_grab
 			<< "grid_padding_grab" << m_cparam->grid_padding_grab
+			<< "offset_grab" << m_cparam->offset_grab
 
 			<< "model_path_cut" << m_cparam->model_path_cut
 			<< "object_threshold_cut" << m_cparam->object_threshold_cut
@@ -57,6 +58,7 @@ namespace graft_cv{
 		m_cparam->grid_row_grab = (int)node["grid_row_grab"];
 		m_cparam->grid_col_grab	= (int)node["grid_col_grab"];
 		m_cparam->grid_padding_grab	= (int)node["grid_padding_grab"];
+		m_cparam->offset_grab = (int)node["offset_grab"];
 		
 		m_cparam->model_path_cut = (string)node["model_path_cut"];
 		m_cparam->object_threshold_cut = (float)node["object_threshold_cut"];
@@ -85,7 +87,9 @@ namespace graft_cv{
 			<< "grid_row_grab:\t" << m_cparam->grid_row_grab << endl
 			<< "grid_col_grab:\t" << m_cparam->grid_col_grab << endl
 			<< "grid_padding_grab:\t" << m_cparam->grid_padding_grab << endl
-			
+			<< "offset_grab:\t" << m_cparam->offset_grab << endl
+
+
 			<< "model_path_cut:\t" << m_cparam->model_path_cut << endl
 			<< "object_threshold_cut:\t" << m_cparam->object_threshold_cut << endl
 			<< "nms_threshold_cut:\t" << m_cparam->nms_threshold_cut << endl

data_def_api.h

@@ -24,6 +24,7 @@ typedef struct{
 	int grid_row_grab;
 	int grid_col_grab;
 	int grid_padding_grab;
+	int offset_grab; //抓取点相对参考点（标注的点5）的偏移像素数
 
 	std::string model_path_cut;
 	float object_threshold_cut;

tcv_conf.yml

@@ -7,10 +7,11 @@ conf_parameters:
    image_backup_days: 7
    model_path_grab: "D:/projects/graft/py_code/retina_tea5/TeaDetector_op9.onnx"    
    object_threshold_grab: 0.45
-   nms_threshold_grab: 0.5
-   grid_row_grab: 3
-   grid_col_grab: 2
-   grid_padding_grab: 50
+   nms_threshold_grab: 0.1
+   grid_row_grab: 2
+   grid_col_grab: 3
+   grid_padding_grab: 100
+   offset_grab: 10
 
    model_path_cut: "D:/projects/graft/py_code/retina_tea5/TeaDetector_op9.onnx"
    object_threshold_cut: 0.45

tea_cv_api.cpp

@@ -5,9 +5,6 @@
 #include "data_def.h"
 #include "data_def_api.h"
 #include "config.h"
-//#include "optimal_angle.h"
-//#include "cut_point_rs.h"
-//#include "cut_point_sc.h"
 #include "logger.h"
 #include "utils.h"
 #include "imstorage_manager.h"
@@ -21,7 +18,7 @@ extern CRITICAL_SECTION g_cs;
 namespace graft_cv
 {
 
-	char *g_version_str = "0.1.3";
+	char *g_version_str = "0.1.5";
 
 	//configure
 	string g_conf_file = "./tcv_conf.yml";	

tea_sorter.cpp

@@ -53,6 +53,10 @@ int CTeaSort::detect(
 
 	//3 load data
 	load_data(imginfo, fn);
+	if (m_cp.image_show) {
+		cv::destroyAllWindows();
+		imshow_wait("input_img", m_raw_img);
+	}
 
 	//4 generate_detect_windows(vector<Rect>&boxes)
 	vector<Rect> drop_regions;
@@ -68,10 +72,50 @@ int CTeaSort::detect(
 		bufftmp << m_imgId << m_dtype_str << "tea detect image regions' size = "<<region_cnt;
 		m_pLogger->INFO(bufftmp.str());
 	}
+	if (m_cp.image_show) {
+		cv::Mat rects_img = m_raw_img.clone();
+		int step_c = int(255 / (float)region_cnt);
+		int step_cc = step_c / 2;
+		int step_ccc = step_cc / 2;
+		int cnt = 0;
+		for (auto&r : drop_regions) {
+			cv::rectangle(rects_img, r, cv::Scalar(step_cc*cnt, step_c*cnt, step_ccc*cnt), 3);
+			cnt += 1;
+		}
+		imshow_wait("regions_img", rects_img);
+	}
 
 	//5 detect	
 	vector<Bbox> droplets_raw;
-	for (auto rect : drop_regions) {
+	int dn = detect_impl(m_raw_img, drop_regions, droplets_raw);
+	if (dn < 2 && m_dtype == img_type::tea_grab) {
+		//up-down flip
+		cv::Mat flip_img;
+		cv::flip(m_raw_img, flip_img, 0);
+		if (m_cp.image_show) {			
+			imshow_wait("flip_img", flip_img);
+		}
+
+		vector<Bbox> droplets_flip;
+		int dn_flip = detect_impl(flip_img, drop_regions, droplets_flip);
+		for (auto&b: droplets_flip) {			
+			int y2 = flip_img.rows - b.y1;
+			int y1 = flip_img.rows - b.y2;
+			b.y1 = y1;
+			b.y2 = y2;
+			
+			for (int i = 0; i < 5; ++i) {				
+				b.ppoint[2 * i + 1] = flip_img.rows - b.ppoint[2 * i + 1];
+			}
+		}
+		if (dn_flip > 0) {
+			droplets_raw.insert(
+				droplets_raw.end(),
+				droplets_flip.begin(),
+				droplets_flip.end());
+		}
+	}
+	/*for (auto rect : drop_regions) {
 		Mat roi = m_raw_img(rect);
 		vector<Bbox> head_droplets = m_drop_detector.RunModel(roi, m_pLogger);
 		if (m_pLogger) {
@@ -98,7 +142,7 @@ int CTeaSort::detect(
 				head_droplets.begin(),
 				head_droplets.end());
 		}
-	}
+	}*/
 	if (m_pLogger) {		
 		stringstream buff_;
 		buff_ << m_imgId<<m_dtype_str << "image detect over. tea number is " << droplets_raw.size();
@@ -123,30 +167,33 @@ int CTeaSort::detect(
 	for (int i = 0; i < droplets.size();++i) {
 		if (i > 1) { break; }
 		Bbox&b = droplets.at(i);
-		double angle = calalate_angle(b);
+		double grab_x, grab_y;
+		double angle = calalate_angle(b, grab_x, grab_y);
 		valid_cnt += 1;
 		//grab point 
 		if (i == 0) {
 			if (m_dtype == img_type::tea_grab) {
-				posinfo.tea_grab_x1 = b.ppoint[8];
-				posinfo.tea_grab_y1 = b.ppoint[9];
+				posinfo.tea_grab_x1 = grab_x;
+				posinfo.tea_grab_y1 = grab_y;
 				posinfo.tea_grab_angle1 = angle;
 			}
 			else {
-				posinfo.tea_cut_x1 = b.ppoint[6];
-				posinfo.tea_cut_y1 = b.ppoint[7];
+				// 切割点是3、4的中间的点
+				posinfo.tea_cut_x1 = 0.5 * (b.ppoint[4] + b.ppoint[6]);
+				posinfo.tea_cut_y1 = 0.5 * (b.ppoint[5] + b.ppoint[7]);
 				posinfo.tea_cut_angle1 = angle;
 			}			
 		}
 		else {
 			if (m_dtype == img_type::tea_grab) {
-				posinfo.tea_grab_x2 = b.ppoint[8];
-				posinfo.tea_grab_y2 = b.ppoint[9];
+				posinfo.tea_grab_x2 = grab_x;
+				posinfo.tea_grab_y2 = grab_y;
 				posinfo.tea_grab_angle2 = angle;
 			}
 			else {
-				posinfo.tea_cut_x2 = b.ppoint[6];
-				posinfo.tea_cut_y2 = b.ppoint[7];
+				// 切割点是3、4的中间的点
+				posinfo.tea_cut_x2 = 0.5 * (b.ppoint[4] + b.ppoint[6]);
+				posinfo.tea_cut_y2 = 0.5 * (b.ppoint[5] + b.ppoint[7]);
 				posinfo.tea_cut_angle2 = angle;
 			}
 
@@ -158,32 +205,75 @@ int CTeaSort::detect(
 		this->clear_imginfo();
 		cv::Mat img_rst = m_raw_img.clone();
 		int cnt = 0;
-		for (auto& b : droplets) {
+		for (auto& b : droplets) {			
+			//rectangle
+			cv::Rect r = cv::Rect(cv::Point2i(b.x1, b.y1), cv::Point2i(b.x2, b.y2));
+			if (cnt < 2) {
+				cv::rectangle(img_rst, r, cv::Scalar(0, 0, 255),2);
+			}
+			else {
+				cv::rectangle(img_rst, r, cv::Scalar(0, 255, 0),2);
+			}
+			//score
 			char name[256];
-			cv::Scalar color(20, 0, 0);//bgr
+			cv::Scalar color(120, 120, 0);//bgr
 			
 			sprintf_s(name, "%.2f", b.score);
 			cv::putText(img_rst, name,
 				cv::Point(b.x1, b.y1),
 				cv::FONT_HERSHEY_COMPLEX, 0.7, color, 2);
 
-			cv::Rect r = cv::Rect(cv::Point2i(b.x1, b.y1), cv::Point2i(b.x2, b.y2));
-			if (cnt < 2) {
-				cv::rectangle(img_rst, r, cv::Scalar(0, 0, 255));
+			//points
+			cv::circle(img_rst, cv::Point(int(b.ppoint[0]), int(b.ppoint[1])), 4, cv::Scalar(255, 0, 255), -1, 3, 0);
+			cv::circle(img_rst, cv::Point(int(b.ppoint[2]), int(b.ppoint[3])), 4, cv::Scalar(0, 255, 255), -1, 3, 0);
+			cv::circle(img_rst, cv::Point(int(b.ppoint[4]), int(b.ppoint[5])), 4, cv::Scalar(255, 0, 0), -1, 3, 0);
+			cv::circle(img_rst, cv::Point(int(b.ppoint[6]), int(b.ppoint[7])), 4, cv::Scalar(0, 255, 0), -1, 3, 0);
+			cv::circle(img_rst, cv::Point(int(b.ppoint[8]), int(b.ppoint[9])), 4, cv::Scalar(0, 0, 255), -1, 3, 0);
+			
+			//grab points
+			if (m_dtype == img_type::tea_grab) {
+				double grab_x, grab_y;
+				calalate_angle(b, grab_x, grab_y);
+				//cv::circle(img_rst, cv::Point(int(grab_x), int(grab_y)), 4, cv::Scalar(0, 215, 255), -1, 3, 0);
+				//lines, p4-p5, p5-grab
+				cv::line(img_rst, 
+					cv::Point(int(b.ppoint[6]), int(b.ppoint[7])), 
+					cv::Point(int(b.ppoint[8]), int(b.ppoint[9])),
+					cv::Scalar(0, 215, 255), 2);
+				cv::line(img_rst, 
+					cv::Point(int(b.ppoint[8]), int(b.ppoint[9])), 
+					cv::Point(int(grab_x), int(grab_y)),
+					cv::Scalar(0, 215, 255), 2);
+				//line x
+				int radius = 20;
+				int cx = int(grab_x);
+				int cy = int(grab_y);
+				cv::line(img_rst, cv::Point(cx - radius, cy - radius), cv::Point(cx + radius, cy + radius), cv::Scalar(0, 215, 255), 2);
+				cv::line(img_rst, cv::Point(cx - radius, cy + radius), cv::Point(cx + radius, cy - radius), cv::Scalar(0, 215, 255), 2);
+
 			}
-			else {
-				cv::rectangle(img_rst, r, cv::Scalar(0, 255, 0));
+			//cut points
+			if (m_dtype == img_type::tea_cut) {	
+				//lines, p3-p4
+				cv::line(img_rst,
+					cv::Point(int(b.ppoint[4]), int(b.ppoint[5])),
+					cv::Point(int(b.ppoint[6]), int(b.ppoint[7])),					
+					cv::Scalar(0, 215, 255), 2);
+				//line x
+				int cx = int(0.5 * (b.ppoint[4] + b.ppoint[6]));
+				int cy = int(0.5 * (b.ppoint[5] + b.ppoint[7]));
+				int radius = 20;
+				cv::line(img_rst, cv::Point(cx - radius, cy - radius), cv::Point(cx + radius, cy + radius), cv::Scalar(0, 215, 255),2);
+				cv::line(img_rst, cv::Point(cx - radius, cy + radius), cv::Point(cx + radius, cy - radius), cv::Scalar(0, 215, 255),2);
 			}
-
-			cv::rectangle(img_rst, r, cv::Scalar(0, 0, 255));
-			cv::circle(img_rst, cv::Point(int(b.ppoint[0]), int(b.ppoint[1])), 4, cv::Scalar(0, 0, 255), -1, 8, 0);
-			cv::circle(img_rst, cv::Point(int(b.ppoint[2]), int(b.ppoint[3])), 4, cv::Scalar(0, 255, 255), -1, 8, 0);
-			cv::circle(img_rst, cv::Point(int(b.ppoint[4]), int(b.ppoint[5])), 4, cv::Scalar(255, 0, 255), -1, 8, 0);
-			cv::circle(img_rst, cv::Point(int(b.ppoint[6]), int(b.ppoint[7])), 4, cv::Scalar(0, 255, 0), -1, 8, 0);
-			cv::circle(img_rst, cv::Point(int(b.ppoint[8]), int(b.ppoint[9])), 4, cv::Scalar(255, 0, 0), -1, 8, 0);
+			
+			
 			cnt += 1;
 
 		}
+		if (m_cp.image_show) {
+			imshow_wait("result_img", img_rst);
+		}
 		m_pImginfoRaw = mat2imginfo(m_raw_img);
 		m_pImginfoDetected = mat2imginfo(img_rst);
 		posinfo.pp_images[0] = m_pImginfoRaw;
@@ -198,7 +288,52 @@ int CTeaSort::detect(
 	return 0;
 }
 
-double CTeaSort::calalate_angle(Bbox&b) {
+int CTeaSort::detect_impl(
+	cv::Mat& raw_img,				//input, image
+	vector<Rect>&drop_regions,	//input, detect regions
+	vector<Bbox> &droplets_raw	//output, detect result
+)
+{
+	//return number of detect result
+	droplets_raw.clear();
+	for (auto rect : drop_regions) {
+		Mat roi = raw_img(rect);
+		vector<Bbox> head_droplets = m_drop_detector.RunModel(roi, m_pLogger);
+		if (m_pLogger) {
+			stringstream buff_;
+			buff_ << m_imgId << m_dtype_str << "-------crop_rect[" << rect.x << "," << rect.y << "," << rect.width
+				<< "," << rect.height << "],"
+				<< " roi image detect over. tea number is " << head_droplets.size();
+			m_pLogger->INFO(buff_.str());
+		}
+		for (Bbox& b : head_droplets) {
+			b.x1 += rect.x;
+			b.x2 += rect.x;
+			b.y1 += rect.y;
+			b.y2 += rect.y;
+			for (int i = 0; i < 5; ++i) {
+				b.ppoint[2 * i] += rect.x;
+				b.ppoint[2 * i + 1] += rect.y;
+			}
+		}
+
+		if (head_droplets.size()) {
+			droplets_raw.insert(
+				droplets_raw.end(),
+				head_droplets.begin(),
+				head_droplets.end());
+		}
+	}
+
+	return droplets_raw.size();
+}
+double CTeaSort::calalate_angle(
+	Bbox&b,				//input
+	double& grab_x,		//output
+	double& grab_y		//output
+)
+{
+	grab_x = grab_y = 0.0;
 	double angle = 0.0;
 	float x3,y3,x4,y4,x5,y5;
 	x3 = b.ppoint[4];
@@ -207,13 +342,27 @@ double CTeaSort::calalate_angle(Bbox&b) {
 	y4 = b.ppoint[7];
 	x5 = b.ppoint[8];
 	y5 = b.ppoint[9];
-	double r45 = sqrt((x4 - x5)*(x4 - x5) + (y4 - y5)*(y4 - y5));
-	if (r45 < 15.0) {
-		angle = atan2(x5 - x3, y5 - y3);
+	if (m_dtype == img_type::tea_grab) {
+		//calculate line of p4 ans p5
+		double r45 = sqrt((x4 - x5)*(x4 - x5) + (y4 - y5)*(y4 - y5));
+		if (r45 < 15.0) {
+			angle = atan2(x5 - x3, y5 - y3);
+		}
+		else {
+			angle = atan2(x5 - x4, y5 - y4);
+		}
+		//计算抓取点
+		double pr = (double)m_cp.offset_grab;
+		double dx = pr * sin(angle);
+		double dy = pr * cos(angle);
+		grab_x = x5 + dx;
+		grab_y = y5 + dy;
 	}
 	else {
-		angle = atan2(x5 - x4, y5 - y4);
+		//tea cut, calculate line of p3 ans p4
+		angle = atan2(x3 - x4, y3 - y4);
 	}
+	
 	angle *= (180.0 / 3.1415926);
 	return angle;
 
@@ -242,14 +391,24 @@ int CTeaSort::load_data(
 				<< "\theight=" << imginfo->height;
 			m_pLogger->INFO(buff.str());
 		}
-		if (!isvalid(imginfo)) {
+		if (!isvalid(imginfo) || (imginfo->channel!=1 && imginfo->channel!=3)) {
 			if (m_pLogger) {
 				m_pLogger->ERRORINFO(m_imgId + m_dtype_str + "input image invalid.");
 			}
 			throw_msg(m_imgId + " invalid input image");
 
 		}
-		m_raw_img = imginfo2mat(imginfo);
+		if (imginfo->channel == 1) {
+			cv::Mat tmp_img = imginfo2mat(imginfo);
+
+			vector<Mat> channels;
+			for (size_t i = 0; i < 3; ++i) { channels.push_back(tmp_img); }			
+			cv::merge(channels, m_raw_img);
+		}
+		else {
+			m_raw_img = imginfo2mat(imginfo);
+		}
+		
 	}
 	else {
 		cv::Mat img = imread(fn, cv::IMREAD_COLOR);

tea_sorter.h

@@ -41,7 +41,8 @@ namespace graft_cv{
 		ImgInfo* m_pImginfoDetected; 
 
 		int generate_detect_windows(vector<Rect>&boxes);
+		int detect_impl(cv::Mat& img, vector<Rect>&drop_regions, vector<Bbox> &droplets_raw);
 		void clear_imginfo();
-		double calalate_angle(Bbox&b);
+		double calalate_angle(Bbox&b, double& grab_x, double&grab_y);
 	};
 };

utils.cpp

@@ -57,7 +57,7 @@ namespace graft_cv{
 	{
 		if(!imginfo){return false;}
 		if(!imginfo->data){return false;}
-		if(imginfo->height*imginfo->width<=0){return false;}
+		if(imginfo->height*imginfo->width<=0){return false;}		
 		return true;
 	}