v0.1.26 抓取功能优化：抓取位置附近评估是否可抓

ReadMe.txt

@@ -23,4 +23,5 @@ v0.1.21
 v0.1.22 切割模型标注关键点更改，2、3点分别是第一、二叶子分叉点
 v0.1.23 抓取点以关键点4为基础偏移
 v0.1.24 切割位置识别,增加kp3_weight_cut参数，控制抓取点在点3和点2间的位置，在[0,1.0]间，越大越靠近点3
-v0.1.25 抓取功能扩展：增加是否空盘识别；增加叶尖朝向优先功能；增加单株优先功能；完善抓取接口返回数值定义
+v0.1.25 抓取功能扩展：增加是否空盘识别；增加叶尖朝向优先功能；增加单株优先功能；完善抓取接口返回数值定义
+v0.1.26 抓取功能优化：抓取位置附近评估是否可抓

inference.cpp

@@ -0,0 +1,185 @@
+#include "inference.h"
+
+//Inference::Inference(const std::string &onnxModelPath, const cv::Size2f &modelInputShape, const std::string &classesTxtFile, const bool &runWithCuda)
+//{
+//    modelPath = onnxModelPath;
+//    modelShape = modelInputShape;
+//    classesPath = classesTxtFile;
+//    cudaEnabled = runWithCuda;
+//
+//    loadOnnxNetwork();
+//    loadClassesFromFile();
+//}
+//
+//std::vector<Detection> Inference::runInference(const cv::Mat &input)
+//{
+//    cv::Mat modelInput = input;
+//    if (letterBoxForSquare && modelShape.width == modelShape.height)
+//        modelInput = formatToSquare(modelInput);
+//
+//    cv::Mat blob;
+//    cv::dnn::blobFromImage(modelInput, blob, 1.0/255.0, modelShape, cv::Scalar(), true, false);
+//    net.setInput(blob);
+//
+//    std::vector<cv::Mat> outputs;
+//    net.forward(outputs, net.getUnconnectedOutLayersNames());
+//
+//    int rows = outputs[0].size[1];
+//    int dimensions = outputs[0].size[2];
+//
+//    bool yolov8 = false;
+//    // yolov5 has an output of shape (batchSize, 25200, 85) (Num classes + box[x,y,w,h] + confidence[c])
+//    // yolov8 has an output of shape (batchSize, 84,  8400) (Num classes + box[x,y,w,h])
+//    if (dimensions > rows) // Check if the shape[2] is more than shape[1] (yolov8)
+//    {
+//        yolov8 = true;
+//        rows = outputs[0].size[2];
+//        dimensions = outputs[0].size[1];
+//
+//        outputs[0] = outputs[0].reshape(1, dimensions);
+//        cv::transpose(outputs[0], outputs[0]);
+//    }
+//    float *data = (float *)outputs[0].data;
+//
+//    float x_factor = modelInput.cols / modelShape.width;
+//    float y_factor = modelInput.rows / modelShape.height;
+//
+//    std::vector<int> class_ids;
+//    std::vector<float> confidences;
+//    std::vector<cv::Rect> boxes;
+//
+//    for (int i = 0; i < rows; ++i)
+//    {
+//        if (yolov8)
+//        {
+//            float *classes_scores = data+4;
+//
+//            cv::Mat scores(1, classes.size(), CV_32FC1, classes_scores);
+//            cv::Point class_id;
+//            double maxClassScore;
+//
+//            minMaxLoc(scores, 0, &maxClassScore, 0, &class_id);
+//
+//            if (maxClassScore > modelScoreThreshold)
+//            {
+//                confidences.push_back(maxClassScore);
+//                class_ids.push_back(class_id.x);
+//
+//                float x = data[0];
+//                float y = data[1];
+//                float w = data[2];
+//                float h = data[3];
+//
+//                int left = int((x - 0.5 * w) * x_factor);
+//                int top = int((y - 0.5 * h) * y_factor);
+//
+//                int width = int(w * x_factor);
+//                int height = int(h * y_factor);
+//
+//                boxes.push_back(cv::Rect(left, top, width, height));
+//            }
+//        }
+//        else // yolov5
+//        {
+//            float confidence = data[4];
+//
+//            if (confidence >= modelConfidenseThreshold)
+//            {
+//                float *classes_scores = data+5;
+//
+//                cv::Mat scores(1, classes.size(), CV_32FC1, classes_scores);
+//                cv::Point class_id;
+//                double max_class_score;
+//
+//                minMaxLoc(scores, 0, &max_class_score, 0, &class_id);
+//
+//                if (max_class_score > modelScoreThreshold)
+//                {
+//                    confidences.push_back(confidence);
+//                    class_ids.push_back(class_id.x);
+//
+//                    float x = data[0];
+//                    float y = data[1];
+//                    float w = data[2];
+//                    float h = data[3];
+//
+//                    int left = int((x - 0.5 * w) * x_factor);
+//                    int top = int((y - 0.5 * h) * y_factor);
+//
+//                    int width = int(w * x_factor);
+//                    int height = int(h * y_factor);
+//
+//                    boxes.push_back(cv::Rect(left, top, width, height));
+//                }
+//            }
+//        }
+//
+//        data += dimensions;
+//    }
+//
+//    std::vector<int> nms_result;
+//    cv::dnn::NMSBoxes(boxes, confidences, modelScoreThreshold, modelNMSThreshold, nms_result);
+//
+//    std::vector<Detection> detections{};
+//    for (unsigned long i = 0; i < nms_result.size(); ++i)
+//    {
+//        int idx = nms_result[i];
+//
+//        Detection result;
+//        result.class_id = class_ids[idx];
+//        result.confidence = confidences[idx];
+//
+//        std::random_device rd;
+//        std::mt19937 gen(rd());
+//        std::uniform_int_distribution<int> dis(100, 255);
+//        result.color = cv::Scalar(dis(gen),
+//                                  dis(gen),
+//                                  dis(gen));
+//
+//        result.className = classes[result.class_id];
+//        result.box = boxes[idx];
+//
+//        detections.push_back(result);
+//    }
+//
+//    return detections;
+//}
+//
+//void Inference::loadClassesFromFile()
+//{
+//    std::ifstream inputFile(classesPath);
+//    if (inputFile.is_open())
+//    {
+//        std::string classLine;
+//        while (std::getline(inputFile, classLine))
+//            classes.push_back(classLine);
+//        inputFile.close();
+//    }
+//}
+//
+//void Inference::loadOnnxNetwork()
+//{
+//    net = cv::dnn::readNetFromONNX(modelPath);
+//    if (cudaEnabled)
+//    {
+//        std::cout << "\nRunning on CUDA" << std::endl;
+//        net.setPreferableBackend(cv::dnn::DNN_BACKEND_CUDA);
+//        net.setPreferableTarget(cv::dnn::DNN_TARGET_CUDA);
+//    }
+//    else
+//    {
+//        std::cout << "\nRunning on CPU" << std::endl;
+//        net.setPreferableBackend(cv::dnn::DNN_BACKEND_OPENCV);
+//        net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU);
+//    }
+//}
+//
+//cv::Mat Inference::formatToSquare(const cv::Mat &source)
+//{
+//    int col = source.cols;
+//    int row = source.rows;
+//    int _max = MAX(col, row);
+//    cv::Mat result = cv::Mat::zeros(_max, _max, CV_8UC3);
+//    source.copyTo(result(cv::Rect(0, 0, col, row)));
+//    return result;
+//}

inference.h

@@ -0,0 +1,51 @@
+#ifndef INFERENCE_H
+#define INFERENCE_H
+
+// Cpp native
+#include <fstream>
+#include <vector>
+#include <string>
+#include <random>
+
+// OpenCV / DNN / Inference
+#include <opencv2/imgproc.hpp>
+#include <opencv2/opencv.hpp>
+#include <opencv2/dnn.hpp>
+
+//struct Detection
+//{
+//    int class_id{0};
+//    std::string className{};
+//    float confidence{0.0};
+//    cv::Scalar color{};
+//    cv::Rect box{};
+//};
+//
+//class Inference
+//{
+//public:
+//    Inference(const std::string &onnxModelPath, const cv::Size2f &modelInputShape, const std::string &classesTxtFile, const bool &runWithCuda = true);
+//    std::vector<Detection> runInference(const cv::Mat &input);
+//
+//private:
+//    void loadClassesFromFile();
+//    void loadOnnxNetwork();
+//    cv::Mat formatToSquare(const cv::Mat &source);
+//
+//    std::string modelPath{};
+//    std::string classesPath{};
+//    bool cudaEnabled{};
+//
+//    std::vector<std::string> classes{};
+//    cv::Size2f modelShape{};
+//
+//    float modelConfidenseThreshold {0.25};
+//    float modelScoreThreshold      {0.45};
+//    float modelNMSThreshold        {0.50};
+//
+//    bool letterBoxForSquare = true;
+//
+//    cv::dnn::Net net;
+//};
+
+#endif // INFERENCE_H

tcv_conf.yml

@@ -6,7 +6,7 @@ conf_parameters:
    image_save: 1
    image_depository: "D:\\logs\\algo_img"
    image_backup_days: 7
-   model_path_grab: "D:/projects/graft/py_code/retina_tea5/TeaDetector_grab_20231223061511.onnx"
+   model_path_grab: "D:/projects/graft/py_code/retina_tea5/TeaDetector_grab_20240106091109.onnx"
    object_threshold_grab: 0.85
    nms_threshold_grab: 1.0000000149011612e-01
    grid_row_grab: 2

tea_cv_api.cpp

@@ -18,7 +18,7 @@ extern CRITICAL_SECTION g_cs;
 namespace graft_cv
 {
 
-	char *g_version_str = "0.1.25";
+	char *g_version_str = "0.1.26";
 
 	//configure
 	string g_conf_file = "./tcv_conf.yml";	

tea_sorter.cpp

@@ -192,9 +192,11 @@ int CTeaSort::detect(
 		pre_cx = -min_dist_grab;
 		pre_cy = -min_dist_grab;
 
-		for (int i = 0; i < droplets.size(); ++i) {
+		for (int i = 0; i < droplets.size(); ++i) {			
 			if (valid_cnt > 1) { break; }
 			Bbox&b = droplets.at(i);
+			if (b.score_overall < m_cp.object_threshold_grab) { continue; }
+
 			double cx = 0.5*(b.x1 + b.x2);
 			double cy = 0.5*(b.y1 + b.y2);
 			double dist = sqrt((cx - pre_cx)*(cx - pre_cx) + (cy - pre_cy)*(cy - pre_cy));
@@ -1678,43 +1680,102 @@ double th/*=50.0*/
 double CTeaSort::singleten_ratio(
 Bbox& box
 )
-{
-	//计算图片中背景的占有率
-	//padding
-	//扩展box的范围，4个方向全部扩展
-	int x1 = box.x1;
-	int y1 = box.y1;
-	int x2 = box.x2;
-	int y2 = box.y2;
+{	//计算苗的方向，找到抓取的位置
+	float x3 = box.ppoint[4];
+	float y3 = box.ppoint[5];
+	float x5 = box.ppoint[8];
+	float y5 = box.ppoint[9];
+	double angle = atan2(x5 - x3, y5 - y3);
+
 	int padding_border = m_cp.offset_grab;
-	x1 -= padding_border;
+
+	float grab_x = x3 + (float)padding_border * sin(angle);
+	float grab_y = y3 + (float)padding_border * cos(angle);
+
+	double singleten_ratio = 0.0;
+	if (grab_x < 0 || grab_y <0 || grab_x> m_raw_img.cols - 1 || grab_y >m_raw_img.rows - 1) {
+		return singleten_ratio;
+	}
+
+	int x1 = int(grab_x) - padding_border/2;
+	int y1 = int(grab_y) - padding_border/2;
+	int x2 = x1 + padding_border;
+	int y2 = y1 + padding_border;
+
 	x1 = x1 < 0 ? 0 : x1;
-	y1 -= padding_border;
 	y1 = y1 < 0 ? 0 : y1;
-
-	x2 += padding_border;
 	x2 = x2 < m_raw_img.cols ? x2 : m_raw_img.cols - 1;
-
-	y2 += padding_border;
 	y2 = y2 < m_raw_img.rows ? y2 : m_raw_img.rows - 1;
-	
-	cv::Rect r(x1,y1,x2-x1,y2-y1);
-	
+
+	cv::Rect r(x1, y1, x2 - x1, y2 - y1);
+
+	//debug
+	if (m_cp.image_show) {
+		cv::Mat tmp = m_raw_img.clone();
+		cv::Rect br(box.x1, box.y1, box.x2 - box.x1, box.y2 - box.y1);
+		cv::rectangle(tmp, br, cv::Scalar(0, 0, 200), 2);
+		cv::rectangle(tmp, r, cv::Scalar(0, 100, 0), 2);
+		imshow_wait("box", tmp);
+	}
+
 	cv::Mat roi = m_raw_gray_img(r).clone();
 	cv::Mat bin_img;
 	double th = cv::threshold(roi, bin_img, 255, 255, cv::THRESH_OTSU);
-	
+
+	if (m_cp.image_show) {		
+		imshow_wait("box bin_img", bin_img);
+	}
+
+
 	//统计bin_img中非0个数
 	double bg_area = 0;
 	cv::Mat_<uchar>::iterator it = bin_img.begin<uchar>();
 	cv::Mat_<uchar>::iterator it_end = bin_img.end<uchar>();
-	for(;it!=it_end;++it){
-		if((*it)>0){
-			bg_area+=1;
+	for (; it != it_end; ++it) {
+		if ((*it)>0) {
+			bg_area += 1;
 		}
 	}
-	double singleten_ratio = bg_area / static_cast<double>(roi.cols * roi.rows);
-	return singleten_ratio;	
+	singleten_ratio = bg_area / static_cast<double>(roi.cols * roi.rows);
+	return singleten_ratio;
+
+
+	////计算图片中背景的占有率
+	////padding
+	////扩展box的范围，4个方向全部扩展
+	//int x1 = box.x1;
+	//int y1 = box.y1;
+	//int x2 = box.x2;
+	//int y2 = box.y2;
+	//int padding_border = m_cp.offset_grab;
+	//x1 -= padding_border;
+	//x1 = x1 < 0 ? 0 : x1;
+	//y1 -= padding_border;
+	//y1 = y1 < 0 ? 0 : y1;
+
+	//x2 += padding_border;
+	//x2 = x2 < m_raw_img.cols ? x2 : m_raw_img.cols - 1;
+
+	//y2 += padding_border;
+	//y2 = y2 < m_raw_img.rows ? y2 : m_raw_img.rows - 1;
+	//
+	//cv::Rect r(x1,y1,x2-x1,y2-y1);
+	//
+	//cv::Mat roi = m_raw_gray_img(r).clone();
+	//cv::Mat bin_img;
+	//double th = cv::threshold(roi, bin_img, 255, 255, cv::THRESH_OTSU);
+	//
+	////统计bin_img中非0个数
+	//double bg_area = 0;
+	//cv::Mat_<uchar>::iterator it = bin_img.begin<uchar>();
+	//cv::Mat_<uchar>::iterator it_end = bin_img.end<uchar>();
+	//for(;it!=it_end;++it){
+	//	if((*it)>0){
+	//		bg_area+=1;
+	//	}
+	//}
+	//double singleten_ratio = bg_area / static_cast<double>(roi.cols * roi.rows);
+	//return singleten_ratio;	
 }
 double CTeaSort::direction_ratio(
 	Bbox& box