v0.1.13 优化抓取角度;优化切割位置

ReadMe.txt

@@ -10,4 +10,5 @@ v0.1.8
 v0.1.9 增加面积大小过滤（通过面积占比系数）
 v0.1.10 优化抓取位置识别（夹角最小点作为抓取点）
 v0.1.11 优化抓取角度
-v0.1.12 优化抓取角度，提高效率，提高准确率
+v0.1.12 优化抓取角度，提高效率，提高准确率
+v0.1.13 优化抓取角度;优化切割位置

data_def.h

@@ -81,12 +81,47 @@ namespace graft_cv{
 
 	struct Bbox //drop box with confidence and center keypoint
 	{
+		Bbox() {
+			this->score = 0.0;
+			this->x1 = 0;
+			this->y1 = 0;
+			this->x2 = 0;
+			this->y2 = 0;
+			for (int i = 0; i < 10; ++i) {
+				this->ppoint[i] = 0.0;
+			}
+			for (int i = 0; i < 2; ++i) {
+				this->operate_point[i] = 0.0;
+			}
+			this->operate_angle = 0.0;
+			this->area = 0.0;
+			this->status = 0;
+		}
+		Bbox(const Bbox& another)
+		{
+			this->score = another.score;
+			this->x1 = another.x1;
+			this->y1 = another.y1;
+			this->x2 = another.x2;
+			this->y2 = another.y2;
+			for (int i = 0; i < 10; ++i) {
+				this->ppoint[i] = another.ppoint[i];
+			}
+			for (int i = 0; i < 2; ++i) {
+				this->operate_point[i] = another.operate_point[i];
+			}
+			this->operate_angle = another.operate_angle;
+			this->area = another.area;
+			this->status = another.status;
+		}
 		float score;
 		int x1;
 		int y1;
 		int x2;
 		int y2;
-		float ppoint[10]; //(x,y) 5 key points	
+		float ppoint[10]; //(x,y) 5 key points
+		float operate_point[2]; //(x,y) 1 operate point, 操作点
+		float operate_angle; //angle of operate point， 操作点的角度方向
 		float area; //像素面积
 		int status;	// 状态:未选中=0；被选中=1；
 	};

tcv_conf.yml

@@ -1,13 +1,13 @@
 %YAML:1.0
 ---
 conf_parameters:
-   image_show: 0
+   image_show: 1
    image_return: 1
    image_save: 1
    image_depository: "D:\\logs\\algo_img"
    image_backup_days: 7
    model_path_grab: "D:/projects/graft/py_code/retina_tea5/TeaDetector_grab_20231205234724.onnx"
-   object_threshold_grab: 0.6
+   object_threshold_grab: 0.65
    nms_threshold_grab: 1.0000000149011612e-01
    grid_row_grab: 2
    grid_col_grab: 3

tea_cv_api.cpp

@@ -18,7 +18,7 @@ extern CRITICAL_SECTION g_cs;
 namespace graft_cv
 {
 
-	char *g_version_str = "0.1.12";
+	char *g_version_str = "0.1.13";
 
 	//configure
 	string g_conf_file = "./tcv_conf.yml";	

tea_detect.cpp

@@ -122,6 +122,10 @@ namespace graft_cv {
 			box.ppoint[8] = rect.keypoints[4].x;
 			box.ppoint[9] = rect.keypoints[4].y;
 
+			box.operate_point[0] = 0.0;
+			box.operate_point[1] = 0.0;
+			box.operate_angle = 0.0;
+
 			box.area = 0.0;
 			box.status = 0;
 			result.push_back(box);

tea_sorter.cpp

@@ -216,9 +216,14 @@ int CTeaSort::detect(
 				posinfo.tea_grab_y2 = grab_y;
 				posinfo.tea_grab_angle2 = angle;
 			}
-			pre_cx = cx;
-			pre_cy = cy;
+
+			b.operate_point[0] = grab_x;
+			b.operate_point[1] = grab_y;
+			b.operate_angle = angle;
 			b.status = 1;
+
+			pre_cx = cx;
+			pre_cy = cy;			
 			valid_cnt += 1;
 		}
 	}
@@ -227,25 +232,29 @@ int CTeaSort::detect(
 		for (int i = 0; i < droplets.size();++i) {
 			if (i > 1) { break; }
 			Bbox&b = droplets.at(i);
-			b.status = 1; // selected
+			
 			double grab_x, grab_y;
 			double angle = calculate_angle(b,/* true,*/ grab_x, grab_y);
 			valid_cnt += 1;			
 			if (i == 0) {				
 				// 切割点是3、4的中间的点
-				posinfo.tea_cut_x1 = 0.5 * (b.ppoint[4] + b.ppoint[6]);
-				posinfo.tea_cut_y1 = 0.5 * (b.ppoint[5] + b.ppoint[7]);
+				posinfo.tea_cut_x1 = grab_x;
+				posinfo.tea_cut_y1 = grab_y;
 				posinfo.tea_cut_angle1 = angle;						
 			}
 			else {				
 				// 切割点是3、4的中间的点
-				posinfo.tea_cut_x2 = 0.5 * (b.ppoint[4] + b.ppoint[6]);
-				posinfo.tea_cut_y2 = 0.5 * (b.ppoint[5] + b.ppoint[7]);
+				posinfo.tea_cut_x2 = grab_x;
+				posinfo.tea_cut_y2 = grab_y;
 				posinfo.tea_cut_angle2 = angle;
-			}		
+			}
+			b.operate_point[0] = grab_x;
+			b.operate_point[1] = grab_y;
+			b.operate_angle = angle;
+			b.status = 1; // selected
+
 		}
-	}
-	
+	}	
 
 	//6 draw
 	if (m_cp.image_return) {
@@ -278,34 +287,40 @@ int CTeaSort::detect(
 			
 			//grab points
 			if (m_dtype == img_type::tea_grab) {
-				double grab_x, grab_y;
-				//bool need_precise = b.status == 1;
-				double grab_angle = calculate_angle(b, /*need_precise,*/ grab_x, grab_y);
-				//cv::circle(img_rst, cv::Point(int(grab_x), int(grab_y)), 4, cv::Scalar(0, 215, 255), -1, 3, 0);
-				//lines, p4-p5, p5-grab
-				cv::line(img_rst, 
-					cv::Point(int(b.ppoint[6]), int(b.ppoint[7])), 
-					cv::Point(int(b.ppoint[8]), int(b.ppoint[9])),
-					cv::Scalar(0, 215, 255), 2);
-				cv::line(img_rst, 
-					cv::Point(int(b.ppoint[8]), int(b.ppoint[9])), 
-					cv::Point(int(grab_x), int(grab_y)),
-					cv::Scalar(0, 215, 255), 2);
-				//line x
-				int radius = 20;
-				int cx = int(grab_x);
-				int cy = int(grab_y);
-				cv::line(img_rst, cv::Point(cx - radius, cy - radius), cv::Point(cx + radius, cy + radius), cv::Scalar(0, 215, 255), 2);
-				cv::line(img_rst, cv::Point(cx - radius, cy + radius), cv::Point(cx + radius, cy - radius), cv::Scalar(0, 215, 255), 2);
-
-				//grab point angle
-				int radius_dir = m_cp.offset_grab / 2;
-				grab_angle *= (CV_PI / 180.0);
-				double dx = radius_dir * sin(grab_angle);
-				double dy = radius_dir * cos(grab_angle);
-				int dir_x = int(grab_x + dx);
-				int dir_y = int(grab_y + dy);
-				cv::line(img_rst, cv::Point(cx, cy), cv::Point(dir_x, dir_y), cv::Scalar(20, 255, 20), 2);
+				if (b.status == 1) {
+					double grab_x, grab_y, grab_angle;
+					grab_x = b.operate_point[0];
+					grab_y = b.operate_point[1];
+					grab_angle = b.operate_angle;
+					//bool need_precise = b.status == 1;
+					//double grab_angle = calculate_angle(b, /*need_precise,*/ grab_x, grab_y);
+					//cv::circle(img_rst, cv::Point(int(grab_x), int(grab_y)), 4, cv::Scalar(0, 215, 255), -1, 3, 0);
+					//lines, p4-p5, p5-grab
+					cv::line(img_rst,
+						cv::Point(int(b.ppoint[6]), int(b.ppoint[7])),
+						cv::Point(int(b.ppoint[8]), int(b.ppoint[9])),
+						cv::Scalar(0, 215, 255), 2);
+					cv::line(img_rst,
+						cv::Point(int(b.ppoint[8]), int(b.ppoint[9])),
+						cv::Point(int(grab_x), int(grab_y)),
+						cv::Scalar(0, 215, 255), 2);
+					//line x
+					int radius = 20;
+					int cx = int(grab_x);
+					int cy = int(grab_y);
+					cv::line(img_rst, cv::Point(cx - radius, cy - radius), cv::Point(cx + radius, cy + radius), cv::Scalar(0, 215, 255), 2);
+					cv::line(img_rst, cv::Point(cx - radius, cy + radius), cv::Point(cx + radius, cy - radius), cv::Scalar(0, 215, 255), 2);
+
+					//grab point angle
+					int radius_dir = m_cp.offset_grab / 2;
+					grab_angle *= (CV_PI / 180.0);
+					double dx = radius_dir * sin(grab_angle);
+					double dy = radius_dir * cos(grab_angle);
+					int dir_x = int(grab_x + dx);
+					int dir_y = int(grab_y + dy);
+					cv::line(img_rst, cv::Point(cx, cy), cv::Point(dir_x, dir_y), cv::Scalar(20, 255, 20), 2);
+				}
+				
 			}
 			//cut points
 			if (m_dtype == img_type::tea_cut) {	
@@ -315,8 +330,8 @@ int CTeaSort::detect(
 					cv::Point(int(b.ppoint[6]), int(b.ppoint[7])),					
 					cv::Scalar(0, 215, 255), 2);
 				//line x
-				int cx = int(0.5 * (b.ppoint[4] + b.ppoint[6]));
-				int cy = int(0.5 * (b.ppoint[5] + b.ppoint[7]));
+				int cx = int(b.operate_point[0]);
+				int cy = int(b.operate_point[1]);
 				int radius = 20;
 				cv::line(img_rst, cv::Point(cx - radius, cy - radius), cv::Point(cx + radius, cy + radius), cv::Scalar(0, 215, 255),2);
 				cv::line(img_rst, cv::Point(cx - radius, cy + radius), cv::Point(cx + radius, cy - radius), cv::Scalar(0, 215, 255),2);
@@ -396,29 +411,22 @@ double CTeaSort::calculate_angle(
 	x5 = b.ppoint[8];
 	y5 = b.ppoint[9];
 	if (m_dtype == img_type::tea_grab) {
-		angle = atan2(x5 - x3, y5 - y3);
-		//if (need_precise_angle) {
-			calculate_stem_grab_position_opt(b, grab_x, grab_y, angle);
-			//计算抓取点
-			if (grab_x < 0 && grab_y < 0) {
-				double pr = (double)m_cp.offset_grab;
-				double dx = pr * sin(angle);
-				double dy = pr * cos(angle);
-				grab_x = x5 + dx;
-				grab_y = y5 + dy;
-			}
-		/*}
-		else {
+		angle = atan2(x5 - x3, y5 - y3);		
+		calculate_stem_grab_position_opt(b, grab_x, grab_y, angle);
+		//计算抓取点
+		if (grab_x < 0 && grab_y < 0) {
 			double pr = (double)m_cp.offset_grab;
 			double dx = pr * sin(angle);
 			double dy = pr * cos(angle);
 			grab_x = x5 + dx;
 			grab_y = y5 + dy;
-		}*/
+		}
+		
 	}
 	else {
 		//tea cut, calculate line of p3 ans p4
 		angle = atan2(x3 - x4, y3 - y4);
+		calculate_stem_cut_position_opt(b, grab_x, grab_y, angle);
 	}
 	
 	angle *= (180.0 / 3.1415926);
@@ -1067,16 +1075,28 @@ void CTeaSort::line_fit(std::vector<cv::Point>& key_point, cv::Vec4f& lines)
 // 2) 重新用局部线性拟合的方向替代ref_angle(原始是p5和p3点连线与y正方向的夹角)
 
 void CTeaSort::calculate_stem_grab_position_opt(
-	Bbox&b,
+	Bbox&b_original,
 	double& grab_x,		//output
 	double& grab_y,		//output
 	double& grab_angle //input-output
 )
 {
+	Bbox b(b_original);
+	int padding_border = m_cp.offset_grab;
+	b.x1 -= padding_border;
+	b.x1 = b.x1 < 0 ? 0 : b.x1;
+	b.y1 -= padding_border;
+	b.y1 = b.y1 < 0 ? 0 : b.y1;
+
+	b.x2 += padding_border;
+	b.x2 = b.x2 < m_raw_img.cols ? b.x2 : m_raw_img.cols - 1;
+
+	b.y2 += padding_border;
+	b.y2 = b.y2 < m_raw_img.rows ? b.y2 : m_raw_img.rows - 1;
 
 	grab_x = grab_y = -1.0;
-	//crop image
-	int padding = 2 * m_cp.offset_grab;
+	//crop image	
+	int padding = 0;
 	int y3 = int(b.ppoint[5]);
 	int y5 = int(b.ppoint[9]);
 	cv::Point p3(int(b.ppoint[4] - b.x1), int(b.ppoint[5] - b.y1));
@@ -1125,7 +1145,8 @@ void CTeaSort::calculate_stem_grab_position_opt(
 		imshow_wait("cropped binary img", bin_img);
 	}
 
-	vector<vector<cv::Point>> contours;
+	//contours image
+	/*vector<vector<cv::Point>> contours;
 	vector<cv::Vec4i> hierarchy;
 	contours.clear();
 	hierarchy.clear();
@@ -1138,7 +1159,16 @@ void CTeaSort::calculate_stem_grab_position_opt(
 	}
 	if (m_cp.image_show) {	
 		imshow_wait("findContours", con_img);
-	}
+	}*/
+
+	// skeletonize() or medial_axis()	
+	cv::Mat ske_img;
+	thinning(bin_img, ske_img);
+	/*if (m_cp.image_show) {
+	imshow_wait("skeleton img", ske_img);
+	}*/
+
+
 	//在grab_angle的指导下找到最优方向，截图，进行局部thinning
 	double ref_angle_init = grab_angle;
 	double delta_angle = CV_PI / 24.0;
@@ -1179,11 +1209,11 @@ void CTeaSort::calculate_stem_grab_position_opt(
 		int pixel_num = 0;
 		for (int r = miny; r <= maxy; ++r) {
 			if (r < 0) { continue; }
-			if (r >= bin_img.rows) { continue; }
+			if (r >= ske_img.rows) { continue; }
 			for (int c = minx; c <= maxx; ++c) {
 				if (c < 0) { continue; }
-				if (c >= bin_img.cols) { continue; }
-				if (con_img.at<unsigned char>(r, c) == 0) { continue; }
+				if (c >= ske_img.cols) { continue; }
+				if (ske_img.at<unsigned char>(r, c) == 0) { continue; }
 
 				double d = cv::pointPolygonTest(triangle_region, cv::Point2f(c, r), false);
 				// d 1-内部点， 0-边缘点 -1-外部点
@@ -1225,7 +1255,7 @@ void CTeaSort::calculate_stem_grab_position_opt(
 
 	//opt box process
 	if (m_cp.image_show) {
-		cv::Mat bin_tmp = bin_img.clone();
+		cv::Mat bin_tmp = ske_img.clone();
 
 		cv::circle(bin_tmp, p5, 4, cv::Scalar(156, 0, 255), 1, 3, 0);
 		cv::circle(bin_tmp, pt0_opt, 4, cv::Scalar(156, 0, 255), 1, 3, 0);
@@ -1243,7 +1273,7 @@ void CTeaSort::calculate_stem_grab_position_opt(
 
 
 	// skeletonize() or medial_axis()	
-	cv::Mat ske_img;
+	/*cv::Mat ske_img;
 	cv::Mat roi_bin_img = cv::Mat::zeros(bin_img.size(), CV_8UC1);	
 	for (int r = miny_opt; r <= maxy_opt; ++r) {
 		if (r < 0) { continue; }
@@ -1259,7 +1289,7 @@ void CTeaSort::calculate_stem_grab_position_opt(
 	thinning(roi_bin_img, ske_img);
 	if (m_cp.image_show) {
 		imshow_wait("skeleton img", ske_img);
-	}
+	}*/
 
 	//通过区域内的骨架点计算ref_angle
 	std::vector<cv::Point> triangle_region;
@@ -1388,4 +1418,137 @@ void CTeaSort::calculate_stem_grab_position_opt(
 	}
 }
 
+void CTeaSort::calculate_stem_cut_position_opt(
+	Bbox&b,
+	double& grab_x,		//output
+	double& grab_y,		//output
+	double& grab_angle //input-output
+)
+{	
+	int padding = 40;
+	
+	grab_x = grab_y = -1.0;
+	//crop image
+
+	cv::Point p3o(int(b.ppoint[4]), int(b.ppoint[5]));
+	cv::Point p4o(int(b.ppoint[6]), int(b.ppoint[7]));
+
+	int x1, y1, x2, y2;
+	x1 = min(p3o.x, p4o.x);
+	y1 = min(p3o.y, p4o.y);
+
+	x2 = max(p3o.x, p4o.x);
+	y2 = max(p3o.y, p4o.y);
+
+	x1 -= padding;
+	x1 = x1 < 0 ? 0 : x1;
+	y1 -= padding;
+	y1 = y1 < 0 ? 0 : y1;
+
+	x2 += padding;
+	x2 = x2 < m_raw_img.cols ?x2 : m_raw_img.cols - 1;
+
+	y2 += padding;
+	y2 = y2 < m_raw_img.rows ? y2 : m_raw_img.rows - 1;
+
+
+	
+	cv::Point p3(int(b.ppoint[4] - x1), int(b.ppoint[5] - y1));
+	cv::Point p4(int(b.ppoint[6] - x1), int(b.ppoint[7] - y1));
+	
+	cv::Mat crop_img;
+	crop_img = m_raw_img(cv::Range(y1, y2), cv::Range(x1, x2)).clone();
+	
+	if (m_cp.image_show) {
+		cv::Mat crop_img_tmp = crop_img.clone();
+		cv::circle(crop_img_tmp, p3, 4, cv::Scalar(255, 0, 0), -1, 3, 0);
+		cv::circle(crop_img_tmp, p4, 4, cv::Scalar(0, 255, 0), -1, 3, 0);
+		imshow_wait("cropped box", crop_img_tmp);
+	}
+
+	//to gray
+	cv::Mat gray_img;
+	if (crop_img.channels() == 1) { gray_img = crop_img; }
+	else {
+		cv::cvtColor(crop_img, gray_img, cv::COLOR_BGR2GRAY);
+	}
+	//binary
+	cv::Mat bin_img;
+	double th = cv::threshold(gray_img, bin_img, 255, 255, cv::THRESH_OTSU);
+	cv::bitwise_not(bin_img, bin_img);
+	if (m_cp.image_show) {
+		imshow_wait("cropped binary img", bin_img);
+	}
+
+	// skeletonize() or medial_axis()	
+	cv::Mat ske_img;
+	thinning(bin_img, ske_img);
+	if (m_cp.image_show) {
+	imshow_wait("skeleton img", ske_img);
+	}
+	
+	cv::Point2f center_pt;
+	center_pt.x = 0.5*(p3.x + p4.x);
+	center_pt.y = 0.5*(p3.y + p4.y);
+
+	int min_x, min_y;
+	double min_loss = 1.0e6;
+
+	double ref_angle = grab_angle + CV_PI / 2.0;
+	if (ref_angle > CV_PI) {
+		ref_angle = ref_angle - 2 * CV_PI;
+	}
+	for (int r = 0; r < ske_img.rows; ++r) {
+		
+		for (int c = 0; c < ske_img.cols; ++c) {			
+			if (ske_img.at<unsigned char>(r, c) == 0) { continue; }
+
+			double target_angle = atan2(double(c- center_pt.x), double(r - center_pt.y));
+			double dangle = intersection_angle(ref_angle, target_angle);
+			if (dangle > CV_PI / 36.0) { continue; }
+
+			double dist = std::powf((center_pt.x - c), 2) + std::powf((center_pt.y - r), 2);
+			dist = std::sqrtf(dist);
+			double loss = dist;
+			// d 1-内部点， 0-边缘点 -1-外部点
+			if (loss < min_loss) {
+				min_loss = loss;
+				min_x = c;
+				min_y = r;
+			}
+		}
+	}
+
+	//另一个方向
+	ref_angle = grab_angle - CV_PI / 2.0;
+	if (ref_angle <  -CV_PI) {
+		ref_angle = ref_angle + 2 * CV_PI;
+	}
+	for (int r = 0; r < ske_img.rows; ++r) {
+
+		for (int c = 0; c < ske_img.cols; ++c) {
+			if (ske_img.at<unsigned char>(r, c) == 0) { continue; }
+
+			double target_angle = atan2(double(c - center_pt.x), double(r - center_pt.y));
+			double dangle = intersection_angle(ref_angle, target_angle);
+			if (dangle > CV_PI / 36.0) { continue; }
+			double dist = std::powf((center_pt.x - c), 2) + std::powf((center_pt.y - r), 2);
+			dist = std::sqrtf(dist);
+			double loss = dist;
+			// d 1-内部点， 0-边缘点 -1-外部点
+			if (loss < min_loss) {
+				min_loss = loss;
+				min_x = c;
+				min_y = r;
+			}
+		}
+	}
+
+	grab_x = min_x;
+	grab_y = min_y;
+
+	grab_x += x1;
+	grab_y += y1;	
+}
+
 }

tea_sorter.h

@@ -46,6 +46,7 @@ namespace graft_cv{
 		double calculate_angle(Bbox&b, /*bool need_precise_angle,*/ double& grab_x, double&grab_y);
 		void calculate_stem_grab_position(Bbox&b, double& grab_x, double&grab_y, double& grab_angle);
 		void calculate_stem_grab_position_opt(Bbox&b, double& grab_x, double&grab_y, double& grab_angle);
+		void calculate_stem_cut_position_opt(Bbox&b, double& grab_x, double&grab_y, double& grab_angle);
 
 		//thinning
 		void thinningIteration(cv::Mat& img, int iter);