v0.1.25 抓取功能扩展：增加是否空盘识别；增加叶尖朝向优先功能；增加单株优先功能；完善抓取接口返回数值定义

ReadMe.txt

@@ -22,4 +22,5 @@ v0.1.20
 v0.1.21 切割图片是彩色图，app提供的通道顺序是rgb，改成opencv用的bgr
 v0.1.22 切割模型标注关键点更改，2、3点分别是第一、二叶子分叉点
 v0.1.23 抓取点以关键点4为基础偏移
-v0.1.24 切割位置识别,增加kp3_weight_cut参数，控制抓取点在点3和点2间的位置，在[0,1.0]间，越大越靠近点3
+v0.1.24 切割位置识别,增加kp3_weight_cut参数，控制抓取点在点3和点2间的位置，在[0,1.0]间，越大越靠近点3
+v0.1.25 抓取功能扩展：增加是否空盘识别；增加叶尖朝向优先功能；增加单株优先功能；完善抓取接口返回数值定义

data_def.h

@@ -83,6 +83,7 @@ namespace graft_cv{
 	{
 		Bbox() {
 			this->score = 0.0;
+			this->score_overall = 0.0;
 			this->x1 = 0;
 			this->y1 = 0;
 			this->x2 = 0;
@@ -100,6 +101,7 @@ namespace graft_cv{
 		Bbox(const Bbox& another)
 		{
 			this->score = another.score;
+			this->score_overall = another.score_overall;
 			this->x1 = another.x1;
 			this->y1 = another.y1;
 			this->x2 = another.x2;
@@ -114,7 +116,8 @@ namespace graft_cv{
 			this->area = another.area;
 			this->status = another.status;
 		}
-		float score;
+		float score;//目标识别得分
+		float score_overall;//识别成功后，优先抓取得分，综合得分
 		int x1;
 		int y1;
 		int x2;

tcv_conf.yml

@@ -7,7 +7,7 @@ conf_parameters:
    image_depository: "D:\\logs\\algo_img"
    image_backup_days: 7
    model_path_grab: "D:/projects/graft/py_code/retina_tea5/TeaDetector_grab_20231223061511.onnx"
-   object_threshold_grab: 0.65
+   object_threshold_grab: 0.85
    nms_threshold_grab: 1.0000000149011612e-01
    grid_row_grab: 2
    grid_col_grab: 3
@@ -15,7 +15,7 @@ conf_parameters:
    offset_grab: 330
    min_distance_grab: 501.
    min_area_ratio_grab: 0.0
-   max_area_ratio_grab: 0.07
+   max_area_ratio_grab: 0.12
    rot_degree_grab: 0
    model_path_cut: "D:/projects/graft/py_code/retina_tea5/TeaDetector_cut_20231219003541.onnx"
    object_threshold_cut: 0.8

tea_cv_api.cpp

@@ -18,7 +18,7 @@ extern CRITICAL_SECTION g_cs;
 namespace graft_cv
 {
 
-	char *g_version_str = "0.1.24";
+	char *g_version_str = "0.1.25";
 
 	//configure
 	string g_conf_file = "./tcv_conf.yml";	
@@ -271,22 +271,22 @@ namespace graft_cv
 			int r = g_tg.load_model();	
 			if (r != 0) {
 				g_logger.ERRORINFO("model load failed");
-				return 1;
+				return 2;
 			}
 			r = g_tg.detect(imginfo, posinfo, fn);
 			return r;						
 		}
 		catch(std::exception &err){
 			g_logger.ERRORINFO(err.what());
-			return 1;
+			return 2;
 		}
 		catch(string& msg){
 			g_logger.ERRORINFO(msg);
-			return 1;
+			return 2;
 		}		
 		catch(...){
 			g_logger.ERRORINFO("unknown error");
-			return 1;
+			return 2;
 		}
 		return 0;
 		

tea_cv_api.h

@@ -83,7 +83,11 @@ namespace graft_cv
 	            // 如果为0，则不存在
 //				posinfo.pp_images;
 //		
-//  返回： 0- 正常，识别到目标； 1- 算法异常失败， -1-没有识别到目标
+//  返回： 
+//      0   - 正常，识别到目标； 
+//      1   - 无算法结果（相机范围内有苗，但是算法没能识别到可以抓取的苗，告诉嵌入式需要抖动），
+//      -1  - 拍照无苗（相机范围内无苗，告诉嵌入式需要上料）
+//       2  - 算法异常失败
 	TCV_API int tea_grab_point(ImgInfo*, PositionInfo& posinfo, const char* fn=0);
 	
 	//12 找到切割位置
@@ -108,7 +112,7 @@ namespace graft_cv
 	// 如果为0，则不存在
 	//				posinfo.pp_images;
 	//		
-	//  返回： 0- 正常，识别到目标； 1- 算法异常失败， -1-没有识别到目标
+	//  返回： 0- 正常，识别到目标； 1- 算法异常失败 或 没有识别到目标
 	TCV_API int tea_cut_point(ImgInfo*, PositionInfo& posinfo, const char* fn = 0);
 
 

tea_sorter.cpp

@@ -85,10 +85,23 @@ int CTeaSort::detect(
 		imshow_wait("regions_img", rects_img);
 	}
 
-	//5 detect	
+	//5 empty feeder dection
+	if (m_dtype == img_type::tea_grab) {
+		bool is_empty = is_empty_feeder(m_raw_gray_img);
+		if (is_empty) {
+			stringstream bufftmp;
+			bufftmp << m_imgId << m_dtype_str << "empty feeder" ;
+			m_pLogger->INFO(bufftmp.str());
+			//拍照无苗, 返回识别结果-1
+			return -1;
+		}
+	}
+	
+
+	//6 detect	
 	vector<Bbox> droplets_raw;
 	int dn = detect_impl(m_raw_img, drop_regions, droplets_raw);
-	if (dn < 2 && m_dtype == img_type::tea_grab) {
+	if (m_dtype == img_type::tea_grab) {
 		//up-down flip
 		cv::Mat flip_img;
 		cv::flip(m_raw_img, flip_img, 0);
@@ -115,43 +128,21 @@ int CTeaSort::detect(
 				droplets_flip.end());
 		}
 	}
-	/*for (auto rect : drop_regions) {
-		Mat roi = m_raw_img(rect);
-		vector<Bbox> head_droplets = m_drop_detector.RunModel(roi, m_pLogger);
-		if (m_pLogger) {
-			stringstream buff_;
-			buff_ << m_imgId << m_dtype_str << "-------crop_rect["<< rect.x<<","<<rect.y<<","<<rect.width
-				<<","<<rect.height<<"],"
-				<<" roi image detect over. tea number is " << head_droplets.size();
-			m_pLogger->INFO(buff_.str());
-		}
-		for (Bbox& b : head_droplets) {
-			b.x1 += rect.x;
-			b.x2 += rect.x;
-			b.y1 += rect.y;
-			b.y2 += rect.y;
-			for (int i = 0; i < 5; ++i) {
-				b.ppoint[2 * i] += rect.x;
-				b.ppoint[2 * i + 1] += rect.y;
-			}			
-		}
-
-		if (head_droplets.size()) {
-			droplets_raw.insert(
-				droplets_raw.end(),
-				head_droplets.begin(),
-				head_droplets.end());
-		}
-	}*/
+	
 	if (m_pLogger) {		
 		stringstream buff_;
 		buff_ << m_imgId<<m_dtype_str << "image detect over. tea number is " << droplets_raw.size();
 		m_pLogger->INFO(buff_.str());
 	}
-	//6 nms, width(height) filt and area calculation
+	//7 nms, width(height) filt and area calculation
 	vector<Bbox> droplets;
 	vector<int> keep;
 	nms_bbox(droplets_raw, m_drop_detector.GetNmsThreshold(), keep);
+	if (m_pLogger) {
+		stringstream buff_;
+		buff_ << m_imgId << m_dtype_str << "after nms_bbox, keep size is " << keep.size();
+		m_pLogger->INFO(buff_.str());
+	}
 	//nms keep and area filter
 	double min_area_th = m_cp.min_area_ratio_grab;
 	double max_area_th = m_cp.max_area_ratio_grab;
@@ -166,7 +157,14 @@ int CTeaSort::detect(
 		area_ratio /= static_cast<double>(m_raw_img.rows);
 		area_ratio /= static_cast<double>(m_raw_img.cols);
 		tbox.area = area_ratio;
-		if (area_ratio < min_area_th || area_ratio > max_area_th) { continue; }
+		if (m_pLogger) {
+			stringstream buff_;
+			buff_ << m_imgId << m_dtype_str << "object's area ratio is " << area_ratio<<", range is ["<< min_area_th<<", "<< max_area_th <<"]";
+			m_pLogger->INFO(buff_.str());
+		}
+		if (area_ratio < min_area_th || area_ratio > max_area_th) { 
+			continue;
+		}
 		//检查box边界是否在图像内，如果没有，修改之
 		if (tbox.x1 < 0) { tbox.x1 = 0; }
 		if (tbox.y1 < 0) { tbox.y1 = 0; }
@@ -188,6 +186,7 @@ int CTeaSort::detect(
 	int valid_cnt = 0;
 	if (m_dtype == img_type::tea_grab) {
 		//grab
+		calculate_overall_score_grab(droplets);//通过综合得分排序
 		double pre_cx, pre_cy;
 		double min_dist_grab = m_cp.min_distance_grab;
 		pre_cx = -min_dist_grab;
@@ -256,7 +255,7 @@ int CTeaSort::detect(
 		}
 	}	
 
-	//6 draw
+	//8 draw
 	if (m_cp.image_return) {
 		this->clear_imginfo();
 		cv::Mat img_rst = m_raw_img.clone();		
@@ -273,7 +272,7 @@ int CTeaSort::detect(
 			char name[256];
 			cv::Scalar color(120, 120, 0);//bgr
 			
-			sprintf_s(name, "%.2f", b.score);
+			sprintf_s(name, "%.2f - %.2f", b.score, b.score_overall);
 			cv::putText(img_rst, name,
 				cv::Point(b.x1, b.y1),
 				cv::FONT_HERSHEY_COMPLEX, 0.7, color, 2);
@@ -350,8 +349,8 @@ int CTeaSort::detect(
 			(*m_ppImgSaver)->saveImage(img_rst, m_imgId + "_rst_0");			
 		}
 	}
-	//拍照无苗, 返回识别结果-1
-	if (valid_cnt == 0) { return -1; }
+	//结果为1无: 算法结果（相机范围内有苗，但是算法没能识别到可以抓取的苗，告诉嵌入式需要抖动）
+	if (valid_cnt == 0) { return 1; }
 	return 0;
 }
 
@@ -522,6 +521,12 @@ int CTeaSort::load_data(
 			m_pLogger->INFO(buff.str());
 		}
 	}
+	//to gray
+	if (m_raw_img.channels() == 1) { m_raw_gray_img = m_raw_img; }
+	else {
+		cv::cvtColor(m_raw_img, m_raw_gray_img, cv::COLOR_BGR2GRAY);
+	}
+	
 	return rst;
 }
 
@@ -1600,4 +1605,145 @@ void CTeaSort::calculate_stem_cut_position_opt(
 	grab_y += y1;	
 }
 
+bool CTeaSort::is_empty_feeder(
+cv::Mat& raw_img,
+double th/*=50.0*/
+)
+{
+	vector<Rect> drop_regions;
+	//生成grid
+	
+	int grid_row = 16;
+	int grid_col = 16;
+	int block_height = int(raw_img.rows / (float)grid_row + 0.5);
+	int block_width = int(raw_img.cols / (float)grid_col + 0.5);
+	for (int r = 0; r < grid_row; ++r) {
+		for (int c = 0; c < grid_col; ++c) {
+			int x0 = c*block_width;
+			int y0 = r*block_height;
+			int x1 = (c + 1)*block_width;
+			int y1 = (r + 1)*block_height;
+
+			if (x0 < 0) { x0 = 0; }
+			if (y0 < 0) { y0 = 0; }
+			if (x1 > raw_img.cols) { x1 = raw_img.cols; }
+			if (y1 > raw_img.rows) { y1 = raw_img.rows; }
+			Rect r(x0, y0, x1 - x0, y1 - y0);
+			drop_regions.push_back(r);
+		}
+	}
+	
+
+	//对原始灰度图进行分析
+	std::vector<double> gray_values;	
+	for (auto rect : drop_regions) {
+		Mat roi = raw_img(rect);
+		cv::Scalar mu = cv::mean(roi);
+		gray_values.push_back(mu[0]);		
+	}
+    bool is_empty = true;
+	double maxv = *max_element(gray_values.begin(), gray_values.end());
+	double minv = *min_element(gray_values.begin(), gray_values.end());	
+	if((maxv-minv)>th){
+		is_empty = false;			
+	}
+	
+	if (is_empty) {
+		return is_empty;
+	}
+
+	//计算前景的百分比
+	cv::Mat bin_img;
+	double th_bin = cv::threshold(raw_img, bin_img, 255, 255, cv::THRESH_OTSU);
+	//统计bin_img中0个数
+	double fg_area = 0;
+	cv::Mat_<uchar>::iterator it = bin_img.begin<uchar>();
+	cv::Mat_<uchar>::iterator it_end = bin_img.end<uchar>();
+	for (; it != it_end; ++it) {
+		if ((*it)==0) {
+			fg_area += 1;
+		}
+	}
+	if (m_cp.image_show) {
+		imshow_wait("overall bin", bin_img);
+	}
+	double objects_ratio = fg_area / static_cast<double>(bin_img.cols * bin_img.rows);
+	if (objects_ratio <= 0.005) {
+		is_empty = true;
+	}
+	return is_empty;
+}
+
+
+double CTeaSort::singleten_ratio(
+Bbox& box
+)
+{
+	//计算图片中背景的占有率
+	//padding
+	//扩展box的范围，4个方向全部扩展
+	int x1 = box.x1;
+	int y1 = box.y1;
+	int x2 = box.x2;
+	int y2 = box.y2;
+	int padding_border = m_cp.offset_grab;
+	x1 -= padding_border;
+	x1 = x1 < 0 ? 0 : x1;
+	y1 -= padding_border;
+	y1 = y1 < 0 ? 0 : y1;
+
+	x2 += padding_border;
+	x2 = x2 < m_raw_img.cols ? x2 : m_raw_img.cols - 1;
+
+	y2 += padding_border;
+	y2 = y2 < m_raw_img.rows ? y2 : m_raw_img.rows - 1;
+	
+	cv::Rect r(x1,y1,x2-x1,y2-y1);
+	
+	cv::Mat roi = m_raw_gray_img(r).clone();
+	cv::Mat bin_img;
+	double th = cv::threshold(roi, bin_img, 255, 255, cv::THRESH_OTSU);
+	
+	//统计bin_img中非0个数
+	double bg_area = 0;
+	cv::Mat_<uchar>::iterator it = bin_img.begin<uchar>();
+	cv::Mat_<uchar>::iterator it_end = bin_img.end<uchar>();
+	for(;it!=it_end;++it){
+		if((*it)>0){
+			bg_area+=1;
+		}
+	}
+	double singleten_ratio = bg_area / static_cast<double>(roi.cols * roi.rows);
+	return singleten_ratio;	
+}
+double CTeaSort::direction_ratio(
+	Bbox& box
+)
+{
+	float x3 = box.ppoint[4];
+	float y3 = box.ppoint[5];	
+	float x5 = box.ppoint[8];
+	float y5 = box.ppoint[9];	
+	double angle = atan2(x5 - x3, y5 - y3);
+	double ratio = cos(angle);
+	if(ratio < 0) {
+		ratio *= -0.75; 
+	}
+	return ratio;
+}
+void CTeaSort::calculate_overall_score_grab(
+	std::vector<Bbox> &boxes
+)
+{
+	for (auto&b : boxes) {
+		double single_ratio = singleten_ratio(b);
+		double dir_score = direction_ratio(b);
+		b.score_overall = single_ratio * dir_score;
+	}
+	sort(boxes.begin(), boxes.end(),
+		[=](const Bbox& left, const Bbox& right) {
+		return left.score_overall > right.score_overall;
+	});
+}
+
 }

tea_sorter.h

@@ -32,6 +32,7 @@ namespace graft_cv{
 		string m_imgId;
 		string m_dtype_str;
 		Mat m_raw_img;
+		Mat m_raw_gray_img;
 		ConfigParam& m_cp;
 		RetinaDrop m_drop_detector;
 		CImStoreManager** m_ppImgSaver;
@@ -76,5 +77,23 @@ namespace graft_cv{
 		计算 [-pi,pi]间的两个角间的夹角
 		*/
 		double intersection_angle(double ref_angle, double angle_to_p3);
+		
+		/**
+		计算灰度图是否为空(料盘是否为空)
+		*/
+		bool is_empty_feeder(cv::Mat& raw_img, double th=20.0);
+		/**
+		计算框图中单例概率
+		*/
+		double singleten_ratio(Bbox& region);
+		/**
+		计算框图方向系数, 针对抓取
+		*/
+		double direction_ratio(Bbox& box);
+
+		/**
+		计算综合得分，并排序, 针对抓取
+		*/
+		void calculate_overall_score_grab(std::vector<Bbox> &drops);
 	};
 };