tea_sorter.cpp 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783
  1. #include <opencv.hpp>
  2. #include <math.h>
  3. #include <io.h>
  4. #include "tea_sorter.h"
  5. #include "utils.h"
  6. using namespace cv;
  7. namespace graft_cv{
  8. CTeaSort::CTeaSort(
  9. ConfigParam& cp,
  10. img_type dtpye,
  11. CGcvLogger*pLog)
  12. :
  13. m_cp(cp),
  14. m_dtype(dtpye),
  15. m_pLogger(pLog),
  16. m_ppImgSaver(0),
  17. m_pImginfoRaw(0),
  18. m_pImginfoDetected(0)
  19. {
  20. m_drop_detector = RetinaDrop(m_pLogger, 0.5, 0.5);
  21. }
  22. CTeaSort::~CTeaSort()
  23. {
  24. clear_imginfo();
  25. }
  26. int CTeaSort::detect(
  27. ImgInfo*imginfo,
  28. PositionInfo& posinfo,
  29. const char* fn
  30. )
  31. {
  32. //1 model status
  33. if (!m_drop_detector.IsModelLoaded()) {
  34. m_pLogger->ERRORINFO(
  35. string("tea detect model NOT loaded"));
  36. return 1;
  37. }
  38. //2 update recognize threshold
  39. if (m_dtype == img_type::tea_grab) {
  40. m_drop_detector.SetThreshold(m_cp.object_threshold_grab, m_cp.nms_threshold_grab);
  41. }
  42. else {
  43. m_drop_detector.SetThreshold(m_cp.object_threshold_cut, m_cp.nms_threshold_cut);
  44. }
  45. //3 load data
  46. load_data(imginfo, fn);
  47. if (m_cp.image_show) {
  48. cv::destroyAllWindows();
  49. imshow_wait("input_img", m_raw_img);
  50. }
  51. //4 generate_detect_windows(vector<Rect>&boxes)
  52. vector<Rect> drop_regions;
  53. int region_cnt = generate_detect_windows(drop_regions);
  54. if (region_cnt == 0) {
  55. stringstream buff_;
  56. buff_ << m_imgId << m_dtype_str << "tea detect image regions' size == 0";
  57. m_pLogger->ERRORINFO(buff_.str());
  58. return 1;
  59. }
  60. else {
  61. stringstream bufftmp;
  62. bufftmp << m_imgId << m_dtype_str << "tea detect image regions' size = "<<region_cnt;
  63. m_pLogger->INFO(bufftmp.str());
  64. }
  65. if (m_cp.image_show) {
  66. cv::Mat rects_img = m_raw_img.clone();
  67. int step_c = int(255 / (float)region_cnt);
  68. int step_cc = step_c / 2;
  69. int step_ccc = step_cc / 2;
  70. int cnt = 0;
  71. for (auto&r : drop_regions) {
  72. cv::rectangle(rects_img, r, cv::Scalar(step_cc*cnt, step_c*cnt, step_ccc*cnt), 3);
  73. cnt += 1;
  74. }
  75. imshow_wait("regions_img", rects_img);
  76. }
  77. //5 detect
  78. vector<Bbox> droplets_raw;
  79. int dn = detect_impl(m_raw_img, drop_regions, droplets_raw);
  80. if (dn < 2 && m_dtype == img_type::tea_grab) {
  81. //up-down flip
  82. cv::Mat flip_img;
  83. cv::flip(m_raw_img, flip_img, 0);
  84. if (m_cp.image_show) {
  85. imshow_wait("flip_img", flip_img);
  86. }
  87. vector<Bbox> droplets_flip;
  88. int dn_flip = detect_impl(flip_img, drop_regions, droplets_flip);
  89. for (auto&b: droplets_flip) {
  90. int y2 = flip_img.rows - b.y1;
  91. int y1 = flip_img.rows - b.y2;
  92. b.y1 = y1;
  93. b.y2 = y2;
  94. for (int i = 0; i < 5; ++i) {
  95. b.ppoint[2 * i + 1] = flip_img.rows - b.ppoint[2 * i + 1];
  96. }
  97. }
  98. if (dn_flip > 0) {
  99. droplets_raw.insert(
  100. droplets_raw.end(),
  101. droplets_flip.begin(),
  102. droplets_flip.end());
  103. }
  104. }
  105. /*for (auto rect : drop_regions) {
  106. Mat roi = m_raw_img(rect);
  107. vector<Bbox> head_droplets = m_drop_detector.RunModel(roi, m_pLogger);
  108. if (m_pLogger) {
  109. stringstream buff_;
  110. buff_ << m_imgId << m_dtype_str << "-------crop_rect["<< rect.x<<","<<rect.y<<","<<rect.width
  111. <<","<<rect.height<<"],"
  112. <<" roi image detect over. tea number is " << head_droplets.size();
  113. m_pLogger->INFO(buff_.str());
  114. }
  115. for (Bbox& b : head_droplets) {
  116. b.x1 += rect.x;
  117. b.x2 += rect.x;
  118. b.y1 += rect.y;
  119. b.y2 += rect.y;
  120. for (int i = 0; i < 5; ++i) {
  121. b.ppoint[2 * i] += rect.x;
  122. b.ppoint[2 * i + 1] += rect.y;
  123. }
  124. }
  125. if (head_droplets.size()) {
  126. droplets_raw.insert(
  127. droplets_raw.end(),
  128. head_droplets.begin(),
  129. head_droplets.end());
  130. }
  131. }*/
  132. if (m_pLogger) {
  133. stringstream buff_;
  134. buff_ << m_imgId<<m_dtype_str << "image detect over. tea number is " << droplets_raw.size();
  135. m_pLogger->INFO(buff_.str());
  136. }
  137. //6 nms, width(height) filt and area calculation
  138. vector<Bbox> droplets;
  139. vector<int> keep;
  140. nms_bbox(droplets_raw, m_drop_detector.GetNmsThreshold(), keep);
  141. //width(height) filter
  142. for (int i : keep) {
  143. droplets.push_back(droplets_raw[i]);
  144. }
  145. if (m_pLogger) {
  146. stringstream buff_;
  147. buff_ << m_imgId << m_dtype_str << "after nms, keep tea number is " << droplets.size();
  148. m_pLogger->INFO(buff_.str());
  149. }
  150. int valid_cnt = 0;
  151. if (m_dtype == img_type::tea_grab) {
  152. //grab
  153. double pre_cx, pre_cy;
  154. double min_dist_grab = m_cp.min_distance_grab;
  155. pre_cx = -min_dist_grab;
  156. pre_cy = -min_dist_grab;
  157. for (int i = 0; i < droplets.size(); ++i) {
  158. if (valid_cnt > 1) { break; }
  159. Bbox&b = droplets.at(i);
  160. double cx = 0.5*(b.x1 + b.x2);
  161. double cy = 0.5*(b.y1 + b.y2);
  162. double dist = sqrt((cx - pre_cx)*(cx - pre_cx) + (cy - pre_cy)*(cy - pre_cy));
  163. if (dist < min_dist_grab) {
  164. continue;
  165. }
  166. double grab_x, grab_y;
  167. double angle = calculate_angle(b, grab_x, grab_y);
  168. //grab point
  169. if (valid_cnt == 0) {
  170. posinfo.tea_grab_x1 = grab_x;
  171. posinfo.tea_grab_y1 = grab_y;
  172. posinfo.tea_grab_angle1 = angle;
  173. }
  174. else {
  175. posinfo.tea_grab_x2 = grab_x;
  176. posinfo.tea_grab_y2 = grab_y;
  177. posinfo.tea_grab_angle2 = angle;
  178. }
  179. pre_cx = cx;
  180. pre_cy = cy;
  181. b.status = 1;
  182. valid_cnt += 1;
  183. }
  184. }
  185. else {
  186. //cut
  187. for (int i = 0; i < droplets.size();++i) {
  188. if (i > 1) { break; }
  189. Bbox&b = droplets.at(i);
  190. b.status = 1; // selected
  191. double grab_x, grab_y;
  192. double angle = calculate_angle(b, grab_x, grab_y);
  193. valid_cnt += 1;
  194. if (i == 0) {
  195. // 切割点是3、4的中间的点
  196. posinfo.tea_cut_x1 = 0.5 * (b.ppoint[4] + b.ppoint[6]);
  197. posinfo.tea_cut_y1 = 0.5 * (b.ppoint[5] + b.ppoint[7]);
  198. posinfo.tea_cut_angle1 = angle;
  199. }
  200. else {
  201. // 切割点是3、4的中间的点
  202. posinfo.tea_cut_x2 = 0.5 * (b.ppoint[4] + b.ppoint[6]);
  203. posinfo.tea_cut_y2 = 0.5 * (b.ppoint[5] + b.ppoint[7]);
  204. posinfo.tea_cut_angle2 = angle;
  205. }
  206. }
  207. }
  208. //6 draw
  209. if (m_cp.image_return) {
  210. this->clear_imginfo();
  211. cv::Mat img_rst = m_raw_img.clone();
  212. for (auto& b : droplets) {
  213. //rectangle
  214. cv::Rect r = cv::Rect(cv::Point2i(b.x1, b.y1), cv::Point2i(b.x2, b.y2));
  215. if (b.status > 0) {
  216. cv::rectangle(img_rst, r, cv::Scalar(0, 0, 255),2);
  217. }
  218. else {
  219. cv::rectangle(img_rst, r, cv::Scalar(0, 255, 0),2);
  220. }
  221. //score
  222. char name[256];
  223. cv::Scalar color(120, 120, 0);//bgr
  224. sprintf_s(name, "%.2f", b.score);
  225. cv::putText(img_rst, name,
  226. cv::Point(b.x1, b.y1),
  227. cv::FONT_HERSHEY_COMPLEX, 0.7, color, 2);
  228. //points
  229. cv::circle(img_rst, cv::Point(int(b.ppoint[0]), int(b.ppoint[1])), 4, cv::Scalar(255, 0, 255), -1, 3, 0);
  230. cv::circle(img_rst, cv::Point(int(b.ppoint[2]), int(b.ppoint[3])), 4, cv::Scalar(0, 255, 255), -1, 3, 0);
  231. cv::circle(img_rst, cv::Point(int(b.ppoint[4]), int(b.ppoint[5])), 4, cv::Scalar(255, 0, 0), -1, 3, 0);
  232. cv::circle(img_rst, cv::Point(int(b.ppoint[6]), int(b.ppoint[7])), 4, cv::Scalar(0, 255, 0), -1, 3, 0);
  233. cv::circle(img_rst, cv::Point(int(b.ppoint[8]), int(b.ppoint[9])), 4, cv::Scalar(0, 0, 255), -1, 3, 0);
  234. //grab points
  235. if (m_dtype == img_type::tea_grab) {
  236. double grab_x, grab_y;
  237. calculate_angle(b, grab_x, grab_y);
  238. //cv::circle(img_rst, cv::Point(int(grab_x), int(grab_y)), 4, cv::Scalar(0, 215, 255), -1, 3, 0);
  239. //lines, p4-p5, p5-grab
  240. cv::line(img_rst,
  241. cv::Point(int(b.ppoint[6]), int(b.ppoint[7])),
  242. cv::Point(int(b.ppoint[8]), int(b.ppoint[9])),
  243. cv::Scalar(0, 215, 255), 2);
  244. cv::line(img_rst,
  245. cv::Point(int(b.ppoint[8]), int(b.ppoint[9])),
  246. cv::Point(int(grab_x), int(grab_y)),
  247. cv::Scalar(0, 215, 255), 2);
  248. //line x
  249. int radius = 20;
  250. int cx = int(grab_x);
  251. int cy = int(grab_y);
  252. cv::line(img_rst, cv::Point(cx - radius, cy - radius), cv::Point(cx + radius, cy + radius), cv::Scalar(0, 215, 255), 2);
  253. cv::line(img_rst, cv::Point(cx - radius, cy + radius), cv::Point(cx + radius, cy - radius), cv::Scalar(0, 215, 255), 2);
  254. }
  255. //cut points
  256. if (m_dtype == img_type::tea_cut) {
  257. //lines, p3-p4
  258. cv::line(img_rst,
  259. cv::Point(int(b.ppoint[4]), int(b.ppoint[5])),
  260. cv::Point(int(b.ppoint[6]), int(b.ppoint[7])),
  261. cv::Scalar(0, 215, 255), 2);
  262. //line x
  263. int cx = int(0.5 * (b.ppoint[4] + b.ppoint[6]));
  264. int cy = int(0.5 * (b.ppoint[5] + b.ppoint[7]));
  265. int radius = 20;
  266. cv::line(img_rst, cv::Point(cx - radius, cy - radius), cv::Point(cx + radius, cy + radius), cv::Scalar(0, 215, 255),2);
  267. cv::line(img_rst, cv::Point(cx - radius, cy + radius), cv::Point(cx + radius, cy - radius), cv::Scalar(0, 215, 255),2);
  268. }
  269. }
  270. if (m_cp.image_show) {
  271. imshow_wait("result_img", img_rst);
  272. }
  273. m_pImginfoRaw = mat2imginfo(m_raw_img);
  274. m_pImginfoDetected = mat2imginfo(img_rst);
  275. posinfo.pp_images[0] = m_pImginfoRaw;
  276. posinfo.pp_images[1] = m_pImginfoDetected;
  277. if (m_ppImgSaver && *m_ppImgSaver) {
  278. (*m_ppImgSaver)->saveImage(img_rst, m_imgId + "_rst_0");
  279. }
  280. }
  281. //拍照无苗, 返回识别结果-1
  282. if (valid_cnt == 0) { return -1; }
  283. return 0;
  284. }
  285. int CTeaSort::detect_impl(
  286. cv::Mat& raw_img, //input, image
  287. vector<Rect>&drop_regions, //input, detect regions
  288. vector<Bbox> &droplets_raw //output, detect result
  289. )
  290. {
  291. //return number of detect result
  292. droplets_raw.clear();
  293. for (auto rect : drop_regions) {
  294. Mat roi = raw_img(rect);
  295. vector<Bbox> head_droplets = m_drop_detector.RunModel(roi, m_pLogger);
  296. if (m_pLogger) {
  297. stringstream buff_;
  298. buff_ << m_imgId << m_dtype_str << "-------crop_rect[" << rect.x << "," << rect.y << "," << rect.width
  299. << "," << rect.height << "],"
  300. << " roi image detect over. tea number is " << head_droplets.size();
  301. m_pLogger->INFO(buff_.str());
  302. }
  303. for (Bbox& b : head_droplets) {
  304. b.x1 += rect.x;
  305. b.x2 += rect.x;
  306. b.y1 += rect.y;
  307. b.y2 += rect.y;
  308. for (int i = 0; i < 5; ++i) {
  309. b.ppoint[2 * i] += rect.x;
  310. b.ppoint[2 * i + 1] += rect.y;
  311. }
  312. }
  313. if (head_droplets.size()) {
  314. droplets_raw.insert(
  315. droplets_raw.end(),
  316. head_droplets.begin(),
  317. head_droplets.end());
  318. }
  319. }
  320. return droplets_raw.size();
  321. }
  322. double CTeaSort::calculate_angle(
  323. Bbox&b, //input
  324. double& grab_x, //output
  325. double& grab_y //output
  326. )
  327. {
  328. grab_x = grab_y = 0.0;
  329. double angle = 0.0;
  330. float x3,y3,x4,y4,x5,y5;
  331. x3 = b.ppoint[4];
  332. y3 = b.ppoint[5];
  333. x4 = b.ppoint[6];
  334. y4 = b.ppoint[7];
  335. x5 = b.ppoint[8];
  336. y5 = b.ppoint[9];
  337. if (m_dtype == img_type::tea_grab) {
  338. calculate_stem_grab_position(b, grab_x, grab_y);
  339. //calculate line of p4 ans p5
  340. double r45 = sqrt((x4 - x5)*(x4 - x5) + (y4 - y5)*(y4 - y5));
  341. if (r45 < 15.0) {
  342. angle = atan2(x5 - x3, y5 - y3);
  343. }
  344. else {
  345. angle = atan2(x5 - x4, y5 - y4);
  346. }
  347. //计算抓取点
  348. if (grab_x < 0 && grab_y < 0) {
  349. double pr = (double)m_cp.offset_grab;
  350. double dx = pr * sin(angle);
  351. double dy = pr * cos(angle);
  352. grab_x = x5 + dx;
  353. grab_y = y5 + dy;
  354. }
  355. }
  356. else {
  357. //tea cut, calculate line of p3 ans p4
  358. angle = atan2(x3 - x4, y3 - y4);
  359. }
  360. angle *= (180.0 / 3.1415926);
  361. return angle;
  362. }
  363. int CTeaSort::load_data(
  364. ImgInfo*imginfo,
  365. const char* fn/* = 0*/)
  366. {
  367. //数据加载功能实现,并生成imageid,保存原始数据到文件
  368. int rst = 0;
  369. //generate image id
  370. if (m_dtype == img_type::tea_grab) {
  371. m_imgId = getImgId(img_type::tea_grab);
  372. m_dtype_str = string(" tea_grab ");
  373. }
  374. else {
  375. m_imgId = getImgId(img_type::tea_cut);
  376. m_dtype_str = string(" tea_cut ");
  377. }
  378. if (imginfo) {
  379. if (m_pLogger) {
  380. stringstream buff;
  381. buff << m_imgId << m_dtype_str << "image, width=" << imginfo->width
  382. << "\theight=" << imginfo->height;
  383. m_pLogger->INFO(buff.str());
  384. }
  385. if (!isvalid(imginfo) || (imginfo->channel!=1 && imginfo->channel!=3)) {
  386. if (m_pLogger) {
  387. m_pLogger->ERRORINFO(m_imgId + m_dtype_str + "input image invalid.");
  388. }
  389. throw_msg(m_imgId + " invalid input image");
  390. }
  391. if (imginfo->channel == 1) {
  392. cv::Mat tmp_img = imginfo2mat(imginfo);
  393. vector<Mat> channels;
  394. for (size_t i = 0; i < 3; ++i) { channels.push_back(tmp_img); }
  395. cv::merge(channels, m_raw_img);
  396. }
  397. else {
  398. m_raw_img = imginfo2mat(imginfo);
  399. }
  400. }
  401. else {
  402. cv::Mat img = imread(fn, cv::IMREAD_COLOR);
  403. if (img.empty()) {
  404. if (m_pLogger) {
  405. m_pLogger->ERRORINFO(m_imgId + m_dtype_str + "input image invalid:" + string(fn));
  406. }
  407. throw_msg(m_imgId + m_dtype_str + "invalid input image: " + string(fn));
  408. }
  409. if (m_pLogger) {
  410. stringstream buff;
  411. buff << m_imgId << m_dtype_str << "image, width=" << img.cols
  412. << "\theight=" << img.rows;
  413. m_pLogger->INFO(buff.str());
  414. }
  415. m_raw_img = img.clone();
  416. }
  417. //image saver
  418. if (m_ppImgSaver && *m_ppImgSaver) {
  419. (*m_ppImgSaver)->saveImage(m_raw_img, m_imgId);
  420. }
  421. return rst;
  422. }
  423. int CTeaSort::load_model()
  424. {
  425. bool b = false;
  426. if (!m_drop_detector.IsModelLoaded()) {
  427. if (m_dtype == img_type::tea_grab) {
  428. b = m_drop_detector.LoadModel(m_cp.model_path_grab);
  429. }
  430. else {
  431. b = m_drop_detector.LoadModel(m_cp.model_path_cut);
  432. }
  433. }
  434. else {
  435. b = true;
  436. }
  437. return b ? 0 : 1;
  438. }
  439. void CTeaSort::clear_imginfo() {
  440. if (m_pImginfoDetected) {
  441. imginfo_release(&m_pImginfoDetected);
  442. m_pImginfoDetected = 0;
  443. }
  444. if (m_pImginfoRaw) {
  445. imginfo_release(&m_pImginfoRaw);
  446. m_pImginfoRaw = 0;
  447. }
  448. }
  449. int CTeaSort::generate_detect_windows(vector<Rect>&boxes)
  450. {
  451. boxes.clear();
  452. int grid_row = m_cp.grid_row_cut;
  453. int grid_col = m_cp.grid_col_cut;
  454. int grid_padding = m_cp.grid_padding_cut;
  455. if (m_dtype == img_type::tea_grab) {
  456. grid_row = m_cp.grid_row_grab;
  457. grid_col = m_cp.grid_col_grab;
  458. grid_padding = m_cp.grid_padding_grab;
  459. }
  460. if (grid_row < 1) { grid_row = 1; }
  461. if (grid_col < 1) { grid_col = 1; }
  462. if (grid_padding < 0) { grid_padding = 0; }
  463. int block_height = int(m_raw_img.rows / (float)grid_row + 0.5);
  464. int block_width = int(m_raw_img.cols / (float)grid_col + 0.5);
  465. for (int r = 0; r < grid_row; ++r) {
  466. for (int c = 0; c < grid_col; ++c) {
  467. int x0 = c*block_width - grid_padding;
  468. int y0 = r*block_height - grid_padding;
  469. int x1 = (c+1)*block_width + grid_padding;
  470. int y1 = (r+1)*block_height + grid_padding;
  471. if (x0 < 0) { x0 = 0; }
  472. if (y0 < 0) { y0 = 0; }
  473. if (x1 > m_raw_img.cols) { x1 = m_raw_img.cols; }
  474. if (y1 > m_raw_img.rows) { y1 = m_raw_img.rows; }
  475. Rect r(x0, y0, x1-x0, y1-y0);
  476. boxes.push_back(r);
  477. }
  478. }
  479. return boxes.size();
  480. }
  481. void CTeaSort::calculate_stem_grab_position(
  482. Bbox&b,
  483. double& grab_x, //output
  484. double& grab_y //output
  485. )
  486. {
  487. grab_x = grab_y = -1.0;
  488. //crop image
  489. int padding = 2 * m_cp.offset_grab;
  490. int y3 = int(b.ppoint[5]);
  491. int y5 = int(b.ppoint[9]);
  492. cv::Point p3(int(b.ppoint[4] - b.x1), int(b.ppoint[5] - b.y1));
  493. cv::Point p5(int(b.ppoint[8] - b.x1), int(b.ppoint[9] - b.y1));
  494. cv::Mat crop_img;
  495. if (y5 > y3) {
  496. // Y position
  497. crop_img = m_raw_img(cv::Range(b.y1, b.y2 + padding), cv::Range(b.x1, b.x2)).clone();
  498. }
  499. else {
  500. // ^ position
  501. if (b.y1 - padding < 0) {
  502. padding = b.y1;
  503. }
  504. p5.y = int(b.ppoint[9] - b.y1 + padding);
  505. p3.y = int(b.ppoint[5] - b.y1 + padding);
  506. crop_img = m_raw_img(cv::Range(b.y1 - padding, b.y2), cv::Range(b.x1, b.x2)).clone();
  507. }
  508. if (m_cp.image_show) {
  509. cv::Mat crop_img_tmp = crop_img.clone();
  510. cv::circle(crop_img_tmp, p5, 4, cv::Scalar(255, 0, 255), -1, 3, 0);
  511. imshow_wait("cropped box", crop_img_tmp);
  512. }
  513. //to gray
  514. cv::Mat gray_img;
  515. if (crop_img.channels() == 1) { gray_img = crop_img; }
  516. else {
  517. cv::cvtColor(crop_img, gray_img, cv::COLOR_BGR2GRAY);
  518. }
  519. //binary
  520. cv::Mat bin_img;
  521. double th = cv::threshold(gray_img, bin_img, 255, 255, cv::THRESH_OTSU);
  522. cv::bitwise_not(bin_img, bin_img);
  523. if (m_cp.image_show) {
  524. imshow_wait("cropped binary img", bin_img);
  525. }
  526. // skeletonize() or medial_axis()
  527. cv::Mat ske_img;
  528. thinning(bin_img, ske_img);
  529. /*if (m_cp.image_show) {
  530. imshow_wait("skeleton img", ske_img);
  531. }*/
  532. //遍历所有点,找到距离等于指定距离的点的位置
  533. std::vector<cv::Point> candidate_pts;
  534. double dist_th = 5;
  535. for (int r = 0; r < ske_img.rows; ++r) {
  536. for (int c = 0; c < ske_img.cols; ++c) {
  537. if (ske_img.at<unsigned char>(r, c) == 0) { continue; }
  538. double dist = std::powf((p5.x - c), 2) + std::powf((p5.y - r),2);
  539. dist = std::sqrtf(dist);
  540. if (std::fabs(dist - m_cp.offset_grab) < dist_th) {
  541. candidate_pts.push_back(cv::Point(c, r));
  542. }
  543. }
  544. }
  545. //按与参考角度的差,找到有效的候选点集合
  546. std::vector<cv::Point> valid_candidate_pts;
  547. double ref_angle = atan2(p5.x - p3.x, p5.y - p3.y);
  548. for (auto&p : candidate_pts) {
  549. double angle_to_p3 = atan2(p.x - p3.x, p.y - p3.y);
  550. //计算夹角
  551. double fabs_angle = 0;
  552. if (ref_angle > 0.5 * CV_PI) {
  553. if (angle_to_p3 < 0) {
  554. angle_to_p3 += 2 * CV_PI;
  555. }
  556. fabs_angle = std::fabs(angle_to_p3 - ref_angle);
  557. }
  558. else {
  559. if (ref_angle < -0.5 * CV_PI) {
  560. if (angle_to_p3 > 0) {
  561. angle_to_p3 -= 2 * CV_PI;
  562. }
  563. fabs_angle = std::fabs(angle_to_p3 - ref_angle);
  564. }
  565. else {
  566. fabs_angle = std::fabs(angle_to_p3 - ref_angle);
  567. }
  568. }
  569. if (fabs_angle > CV_PI / 4.0) { continue; }
  570. valid_candidate_pts.push_back(p);
  571. }
  572. // 找到离重心最近的点作为抓取点
  573. if (valid_candidate_pts.size() > 0) {
  574. cv::Point2f p_mu(0,0);
  575. for (auto&p : valid_candidate_pts) {
  576. p_mu.x += p.x;
  577. p_mu.y += p.y;
  578. }
  579. p_mu.x /= (float)(valid_candidate_pts.size());
  580. p_mu.y /= (float)(valid_candidate_pts.size());
  581. double min_dist = 1.0e8;
  582. for (auto&p : valid_candidate_pts) {
  583. double dist = std::powf((p.x - p_mu.x), 2) + std::powf((p.y - p_mu.y), 2);
  584. dist = std::sqrtf(dist);
  585. if (dist < min_dist) {
  586. min_dist = dist;
  587. grab_x = p.x;
  588. grab_y = p.y;
  589. }
  590. }
  591. }
  592. if (m_cp.image_show) {
  593. cv::Mat ske_img_tmp = ske_img.clone();
  594. for (auto&p : valid_candidate_pts) {
  595. ske_img_tmp.at<unsigned char>(p) = 100;
  596. }
  597. cv::circle(ske_img_tmp, p5, 4, cv::Scalar(255, 0, 255), 1, 3, 0);
  598. if (grab_x > 0 && grab_y > 0) {
  599. cv::circle(ske_img_tmp, cv::Point(int(grab_x), int(grab_y)), 4, cv::Scalar(156, 0, 255), 1, 3, 0);
  600. }
  601. imshow_wait("skeleton img label", ske_img_tmp);
  602. }
  603. //重新得到grab_x,grab_y的坐标
  604. if (grab_x > 0 && grab_y > 0) {
  605. int real_padding_y = p5.y - int(b.ppoint[9] - b.y1);
  606. grab_y -= real_padding_y;
  607. grab_y += b.y1;
  608. grab_x += b.x1;
  609. }
  610. }
  611. /**
  612. * Code for thinning a binary image using Zhang-Suen algorithm.
  613. *
  614. * Author: Nash (nash [at] opencv-code [dot] com)
  615. * Website: http://opencv-code.com
  616. */
  617. /**
  618. * Perform one thinning iteration.
  619. * Normally you wouldn't call this function directly from your code.
  620. *
  621. * Parameters:
  622. * im Binary image with range = [0,1]
  623. * iter 0=even, 1=odd
  624. */
  625. void CTeaSort::thinningIteration(cv::Mat& img, int iter)
  626. {
  627. CV_Assert(img.channels() == 1);
  628. CV_Assert(img.depth() != sizeof(uchar));
  629. CV_Assert(img.rows > 3 && img.cols > 3);
  630. cv::Mat marker = cv::Mat::zeros(img.size(), CV_8UC1);
  631. int nRows = img.rows;
  632. int nCols = img.cols;
  633. if (img.isContinuous()) {
  634. nCols *= nRows;
  635. nRows = 1;
  636. }
  637. int x, y;
  638. uchar *pAbove;
  639. uchar *pCurr;
  640. uchar *pBelow;
  641. uchar *nw, *no, *ne; // north (pAbove)
  642. uchar *we, *me, *ea;
  643. uchar *sw, *so, *se; // south (pBelow)
  644. uchar *pDst;
  645. // initialize row pointers
  646. pAbove = NULL;
  647. pCurr = img.ptr<uchar>(0);
  648. pBelow = img.ptr<uchar>(1);
  649. for (y = 1; y < img.rows - 1; ++y) {
  650. // shift the rows up by one
  651. pAbove = pCurr;
  652. pCurr = pBelow;
  653. pBelow = img.ptr<uchar>(y + 1);
  654. pDst = marker.ptr<uchar>(y);
  655. // initialize col pointers
  656. no = &(pAbove[0]);
  657. ne = &(pAbove[1]);
  658. me = &(pCurr[0]);
  659. ea = &(pCurr[1]);
  660. so = &(pBelow[0]);
  661. se = &(pBelow[1]);
  662. for (x = 1; x < img.cols - 1; ++x) {
  663. // shift col pointers left by one (scan left to right)
  664. nw = no;
  665. no = ne;
  666. ne = &(pAbove[x + 1]);
  667. we = me;
  668. me = ea;
  669. ea = &(pCurr[x + 1]);
  670. sw = so;
  671. so = se;
  672. se = &(pBelow[x + 1]);
  673. int A = (*no == 0 && *ne == 1) + (*ne == 0 && *ea == 1) +
  674. (*ea == 0 && *se == 1) + (*se == 0 && *so == 1) +
  675. (*so == 0 && *sw == 1) + (*sw == 0 && *we == 1) +
  676. (*we == 0 && *nw == 1) + (*nw == 0 && *no == 1);
  677. int B = *no + *ne + *ea + *se + *so + *sw + *we + *nw;
  678. int m1 = iter == 0 ? (*no * *ea * *so) : (*no * *ea * *we);
  679. int m2 = iter == 0 ? (*ea * *so * *we) : (*no * *so * *we);
  680. if (A == 1 && (B >= 2 && B <= 6) && m1 == 0 && m2 == 0)
  681. pDst[x] = 1;
  682. }
  683. }
  684. img &= ~marker;
  685. }
  686. /**
  687. * Function for thinning the given binary image
  688. *
  689. * Parameters:
  690. * src The source image, binary with range = [0,255]
  691. * dst The destination image
  692. */
  693. void CTeaSort::thinning(const cv::Mat& src, cv::Mat& dst)
  694. {
  695. dst = src.clone();
  696. dst /= 255; // convert to binary image
  697. cv::Mat prev = cv::Mat::zeros(dst.size(), CV_8UC1);
  698. cv::Mat diff;
  699. do {
  700. thinningIteration(dst, 0);
  701. thinningIteration(dst, 1);
  702. cv::absdiff(dst, prev, diff);
  703. dst.copyTo(prev);
  704. } while (cv::countNonZero(diff) > 0);
  705. dst *= 255;
  706. }
  707. }