From 2e706e9f8af03f071ed86c30f844a1698de7c59c Mon Sep 17 00:00:00 2001 From: chief Date: Sun, 7 Aug 2022 10:01:56 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B0=83=E6=95=B4=E9=A1=BA=E5=BA=8F=E5=9B=BE?= =?UTF-8?q?=E8=AF=86=E5=88=AB=E9=98=88=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cddetector/BizarreSituations.java | 1 + .../uct/umlrecog/sddetector/LoopDetector.java | 212 ++++++++++++++++++ .../com/hy/java/uct/util/sd/UMLObject.java | 7 + 3 files changed, 220 insertions(+) diff --git a/src/main/java/com/hy/java/uct/umlrecog/cddetector/BizarreSituations.java b/src/main/java/com/hy/java/uct/umlrecog/cddetector/BizarreSituations.java index a1a4682..04c5580 100644 --- a/src/main/java/com/hy/java/uct/umlrecog/cddetector/BizarreSituations.java +++ b/src/main/java/com/hy/java/uct/umlrecog/cddetector/BizarreSituations.java @@ -7,6 +7,7 @@ public class BizarreSituations { public static double rect_containing_tri_to_be_ext = 0.7; public static double rect_not_containing_tri_still_to_be_ext = 0.5; public static double rect_containing_bizaare_to_be_aggr = 0.3; + public static double rect_containing_bizaare = 0.3; public static double getThreshold(double blur_line, double color_too_light, double line_over_class, double gradual_color, double short_line, double too_close, double text_in_line) { // double result = 0.5; diff --git a/src/main/java/com/hy/java/uct/umlrecog/sddetector/LoopDetector.java b/src/main/java/com/hy/java/uct/umlrecog/sddetector/LoopDetector.java index 633507e..df82f19 100644 --- a/src/main/java/com/hy/java/uct/umlrecog/sddetector/LoopDetector.java +++ b/src/main/java/com/hy/java/uct/umlrecog/sddetector/LoopDetector.java @@ -1,5 +1,217 @@ package com.hy.java.uct.umlrecog.sddetector; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.opencv.core.Mat; +import org.opencv.core.MatOfPoint; +import org.opencv.core.MatOfPoint2f; +import org.opencv.core.Point; +import org.opencv.core.Rect; +import org.opencv.core.Scalar; +import org.opencv.core.Size; +import org.opencv.imgcodecs.Imgcodecs; +import org.opencv.imgproc.Imgproc; + +import com.hy.java.uct.umlrecog.cddetector.BizarreSituations; +import com.hy.java.uct.umlrecog.util.ImgProcessor; +import com.hy.java.uct.umlrecog.util.Rectangle; +import com.hy.java.uct.util.sd.Message; +import com.hy.java.uct.util.sd.UMLObject; +import com.hy.java.utility.common.Pair; + public class LoopDetector { + private String sd_path = null; + + /** + * 识别特殊符号 + * + * @param objs_in_sd + * @param dash_lines + * @param solid_lines + * @param dash_lines + * @return + */ + public Pair> detectRelationType(Pair> objs_in_sd, Set solid_lines, Set dash_lines, double ratio) { + System.out.println("开始识别" + sd_path + "中所有符号"); + Mat cls_diagram = objs_in_sd.getLeft(); + Mat origin_cls_diagram = Imgcodecs.imread(sd_path, Imgcodecs.IMREAD_GRAYSCALE); + /* + * 识别图中所有“轮廓”并存在contours中 + * + * detect all "contours" in the image and save in the contours + */ + // 先腐蚀（让符号像素侵蚀周围空间，得以连接起来）、后膨胀（让点周围的空白区域膨胀，从而去掉孤立的噪点），提高识别准确率 + // erosion, then dilation, in order to increase the detection accuracy + Imgproc.erode(cls_diagram, cls_diagram, Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(3.5, 3.5)), new Point(-1, -1), 4); + Imgproc.dilate(cls_diagram, cls_diagram, Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(1.7, 1.7)), new Point(-1, -1), 5); + List contours = new ArrayList<>(); + Imgproc.findContours(cls_diagram, contours, new Mat(), Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE); + /* + * 对每个轮廓contour，检测是否为某种关系符号 + * + * for each contour, check to see if it is a relationship type symbol + */ + // 根据图片像素计算轮廓面积阈值。如果轮廓面积太小或太大，则直接忽略 + // calculating the threshold for contour_area according to the image size. If the contour_area is too big or small, ignore it + long cd_area = cls_diagram.width() * cls_diagram.height(); + double min_area = cd_area * ratio; + double max_area = cd_area * 0.5; + for (MatOfPoint contour : contours) { + // 如果轮廓面积太小或太大，则直接忽略 + // If the contour_area is too big or small, ignore it + double contour_area = Imgproc.contourArea(contour); + if (contour_area < min_area || contour_area > max_area) { + // 将噪点直接涂白 + // wipe the noise points + Imgproc.fillConvexPoly(cls_diagram, contour, new Scalar(255, 255, 255)); + continue; + } + // 如果轮廓面积合适，则用矩形将其包裹，并裁剪原图中的矩形区域、重新识别符号 + // If the contour_area is ok, then make a rectangle to cover it, and cut the rectangle into a new image + MatOfPoint2f curve = new MatOfPoint2f(contour.toArray()); + MatOfPoint2f approx_curve = new MatOfPoint2f(); + Imgproc.approxPolyDP(curve, approx_curve, 0.01 * Imgproc.arcLength(curve, true), true); + // 获取包络矩形 + // get the bounding rect + Rect rect_containing_rela_type = Imgproc.boundingRect(new MatOfPoint(approx_curve.toArray())); + // 如果包络矩形包含某一条关系线的某一端，则检测该矩形内的符号 + // 对于虚线，由于只可能是实现或生命线，所以只需要知道是否包含即可，不用检测矩形内部图形。belong_to_dash_rela为true表示当前关系符号属于某条虚线关系线 + // check the relationship type symbol in the bounding rect + // 裁剪原图中包含关系符号的矩形区域 + // cut the original area containing the relationship type symbol + Mat cutted_origin_rela_type_area = ImgProcessor.cutImage(origin_cls_diagram, rect_containing_rela_type); + Imgproc.erode(cutted_origin_rela_type_area, cutted_origin_rela_type_area, Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(2, 2)), new Point(-1, -1), 2); + Imgproc.dilate(cutted_origin_rela_type_area, cutted_origin_rela_type_area, Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(1, 1)), new Point(-1, -1), 2); + // 针对包含关系符号的区域，重新识别各种“轮廓”，从而发现真正的关系符号 + // re-detect all "contours" to find real relationship type symbol + List origin_contours = new ArrayList<>(); + Imgproc.findContours(cutted_origin_rela_type_area, origin_contours, new Mat(), Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE); + // 针对识别出来的各种“轮廓”做筛选 + // filter the "contours" + for (MatOfPoint origin_contour : origin_contours) { + MatOfPoint2f origin_curve = new MatOfPoint2f(origin_contour.toArray()); + MatOfPoint2f origin_approx_curve = new MatOfPoint2f(); + // 对“轮廓”做二次逼近，获取关系符号形状。第一次逼近可以误差大一点，得到大致形状；第二次逼近误差要小一点，得到精确形状 + // the double-approximation, to get the shape of the relationship type symbol. The first approximation stretches the shape; the second approximation is more delicate + Imgproc.approxPolyDP(origin_curve, origin_approx_curve, 0.05 * Imgproc.arcLength(origin_curve, false), true); + Imgproc.approxPolyDP(origin_approx_curve, origin_approx_curve, 0.01 * Imgproc.arcLength(origin_approx_curve, true), true); + double threshold = BizarreSituations.getThreshold(1, 1, 0, 1, 1, 1, 0); + // double threshold = BizaareSituations.getThreshold(1, 1, 0, 1, 1, 0, 0); + // double threshold = BizaareSituations.getThreshold(0.2, 0.3, 0.5, 0.55, 0.6, 0.1, 0.03); + // if we get a triangle, then it is a generalization + if (origin_approx_curve.toArray().length <= 5) { + // 获取包络矩形 + // get the bounding rect + Rect rect_possible_containing_ext = Imgproc.boundingRect(new MatOfPoint(origin_approx_curve.toArray())); + // 看图是否异常 + // some unexpected situations may influence the recognition + if (threshold <= BizarreSituations.rect_containing_bizaare) { + System.out.println(rect_possible_containing_ext.x + "\t" + rect_possible_containing_ext.y + "\t" + rect_possible_containing_ext.width + "\t" + rect_possible_containing_ext.height); + objs_in_sd.getRight().add(new UMLObject("contains" + rect_possible_containing_ext.toString())); + } + } + } + break; + } + System.out.println("完成对" + sd_path + "中识别符号的识别。"); + return objs_in_sd; + } + /** + * 检测方框 + * + * detect rectangles + * + * @param sd_diagram + * @param ratio 检测最小矩形占全图面积的比例（0~1之间的一个小数）。如果顺序图中对象的面积很小，则该比例应设的很小。 + * @return + */ + public Pair> detectRectArea(Mat sd_diagram, double ratio) { + System.out.println("开始识别" + sd_path + "中所有矩形"); + /* + * 识别图中所有“轮廓”并存在contours中 + * + * recognize all "contours" in the image, and save them in the contours + */ + List contours = new ArrayList<>(); + Imgproc.findContours(sd_diagram, contours, new Mat(), Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE); + /* + * 对每个轮廓contour，检测是否为矩形，并将contour和检测结果存在Rectangle中。所有轮廓的检测结果最终存在all_rect_areas中 + * + * for each contour, check whether it is a rectangle, then save both the contour and the check result in Rectangle. All check results for contours are saved in all_rect_areas + */ + List all_rect_areas = new ArrayList<>(); + // 这个rect_contours里只存那些检测结果为矩形的contour，用于后面边框涂白，防止矩形干扰关系符号和关系线识别 + // only rectangle contours are saved in rect_contours, for wiping, in order to avoid their influence on relationship recognition + List rect_contours = new ArrayList<>(); + // 根据图片像素计算轮廓面积阈值。如果轮廓面积太小或太大，则直接忽略 + // calculating the threshold for contour_area according to the image size. If the contour_area is too big or small, ignore it + long cd_area = sd_diagram.width() * sd_diagram.height(); + double min_cls_area = 60; + double max_cls_area = cd_area * 0.8; + for (MatOfPoint contour : contours) { + // 如果轮廓面积太小或太大，则直接忽略 + // If the contour_area is too big or small, ignore it + double contour_area = Imgproc.contourArea(contour); + if (contour_area < min_cls_area || contour_area > max_cls_area) { + continue; + } + // 如果轮廓面积合适，则检测是否为矩形。采用多边形逼近法，将轮廓转化为curve向多边形做逼近，并将逼近结果存在approx_curve中 + // If the contour_area is ok, then check if it is a rectangle based on polygon approximation + MatOfPoint2f curve = new MatOfPoint2f(contour.toArray()); + MatOfPoint2f approx_curve = new MatOfPoint2f(); + Imgproc.approxPolyDP(curve, approx_curve, 0.01 * Imgproc.arcLength(curve, true), true); + // 针对逼近结果approx_curve，若其共有4个顶点，则可认为是矩形。将检测结果绘制在原图中，便于后续处理 + // if the approximation has 4 vertices, it can be considered as a rectangle + if (approx_curve.toArray().length == 4) { + // 将矩形存在all_rect_areas中用于类的整合和文字识别。存完后将其从原图中抹掉，防止其干扰关系符号和关系线识别 + // save the rectangle in all_rect_areas for merging and text recognition, then wipe it + all_rect_areas.add(new Rectangle(sd_diagram.clone(), contour, approx_curve)); + rect_contours.add(contour); + // 存完后将矩形从图中抹掉（涂白）。后面还需对所有边框进行涂白 + // wipe the rectangle to avoid its influence on relationship recognition + Imgproc.fillConvexPoly(sd_diagram, contour, new Scalar(255, 255, 255)); + } + } + // 对所有矩形边框进行涂白，防止其干扰关系符号和关系线识别 + // wipe the rectangle to avoid its influence on relationship recognition + Imgproc.drawContours(sd_diagram, rect_contours, -1, new Scalar(255, 255, 255), 5); + // 由于灰度问题，此时可能会有遗留矩形，所以重新检测一遍 + // due to the gray-scale problem, there may be other rectangles, so detect them again. all the following process are the same + contours.clear(); + rect_contours.clear(); + Imgproc.findContours(sd_diagram, contours, new Mat(), Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE); + for (MatOfPoint contour : contours) { + // 如果轮廓面积太小或太大，则直接忽略 + double contour_area = Imgproc.contourArea(contour); + if (contour_area < min_cls_area || contour_area > max_cls_area) { + continue; + } + // 如果轮廓面积合适，则检测是否为矩形。采用多边形逼近法，将轮廓转化为curve向多边形做逼近，并将逼近结果存在approx_curve中 + MatOfPoint2f curve = new MatOfPoint2f(contour.toArray()); + MatOfPoint2f approx_curve = new MatOfPoint2f(); + Imgproc.approxPolyDP(curve, approx_curve, 0.05 * Imgproc.arcLength(curve, false), true); + Imgproc.approxPolyDP(approx_curve, approx_curve, 0.01 * Imgproc.arcLength(approx_curve, true), true); + // 针对逼近结果approx_curve，若其共有4个顶点，则可认为是矩形。将检测结果绘制在原图中，便于后续处理 + if (approx_curve.toArray().length == 4) { + // 将矩形存在all_rect_areas中用于类的整合和文字识别。存完后将其从原图中抹掉，防止其干扰关系符号和关系线识别 + all_rect_areas.add(new Rectangle(sd_diagram.clone(), contour, approx_curve)); + rect_contours.add(contour); + // 存完后将矩形从图中抹掉（涂白）。后面还需对所有边框进行涂白 + Imgproc.fillConvexPoly(sd_diagram, contour, new Scalar(255, 255, 255)); + // 可以不涂白，看一下识别结果。最后注释掉 + // Imgproc.fillConvexPoly(cls_diagram, contour, new Scalar(55, 55, 55)); + // Imgcodecs.imwrite(temp_res_path, cls_diagram); + // System.out.println(approx_curve.dump()); + } + } + // 对所有矩形边框进行涂白，防止其干扰关系符号和关系线识别 + Imgproc.drawContours(sd_diagram, rect_contours, -1, new Scalar(255, 255, 255), 5); + // 看一眼，最后注释掉 + // Imgcodecs.imwrite(temp_res_path, cls_diagram); + // 此时的cls_diagram中矩形已涂白 + return Pair.createPair(sd_diagram, all_rect_areas); + } } diff --git a/src/main/java/com/hy/java/uct/util/sd/UMLObject.java b/src/main/java/com/hy/java/uct/util/sd/UMLObject.java index abd7171..39d9eff 100644 --- a/src/main/java/com/hy/java/uct/util/sd/UMLObject.java +++ b/src/main/java/com/hy/java/uct/util/sd/UMLObject.java @@ -19,6 +19,13 @@ public class UMLObject { public List in_msgs = new ArrayList<>(); public List mapped_file_dir_ls = new ArrayList<>(); + public UMLObject(String title) { + setTitle(title); + } + + public UMLObject() { + } + public String getTitle() { return title; } -- Gitee