diff --git "a/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/ONNX\346\250\241\345\236\213\346\216\250\347\220\206\346\214\207\345\257\274/benchmark/cv/segmentation/PointRend.patch" "b/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/ONNX\346\250\241\345\236\213\346\216\250\347\220\206\346\214\207\345\257\274/benchmark/cv/segmentation/PointRend.patch" deleted file mode 100644 index 50a209c738563810911c149705fd87cb0300e6f6..0000000000000000000000000000000000000000 --- "a/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/ONNX\346\250\241\345\236\213\346\216\250\347\220\206\346\214\207\345\257\274/benchmark/cv/segmentation/PointRend.patch" +++ /dev/null @@ -1,171 +0,0 @@ -1.从mmcv复制替换掉grid_sample的方法 -2.去除部分动态shape的where算子 -3.expand输入类型转换 -4.div输入类型转换 -diff --git a/detectron2/modeling/meta_arch/semantic_seg.py b/detectron2/modeling/meta_arch/semantic_seg.py -index 7db8410..8a08e76 100644 ---- a/detectron2/modeling/meta_arch/semantic_seg.py -+++ b/detectron2/modeling/meta_arch/semantic_seg.py -@@ -92,10 +92,9 @@ class SemanticSegmentor(nn.Module): - The prediction has shape KxHxW that represents the logits of - each class for each pixel. - """ -- images = [x["image"].to(self.device) for x in batched_inputs] -+ '''images = [x["image"].to(self.device) for x in batched_inputs] - images = [(x - self.pixel_mean) / self.pixel_std for x in images] - images = ImageList.from_tensors(images, self.backbone.size_divisibility) -- - features = self.backbone(images.tensor) - - if "sem_seg" in batched_inputs[0]: -@@ -104,8 +103,13 @@ class SemanticSegmentor(nn.Module): - targets, self.backbone.size_divisibility, self.sem_seg_head.ignore_value - ).tensor - else: -- targets = None -+ targets = None''' -+ batched_inputs = batched_inputs.to(self.device) -+ raw_input = batched_inputs -+ features = self.backbone(raw_input) -+ targets = None - results, losses = self.sem_seg_head(features, targets) -+ return results - - if self.training: - return losses -diff --git a/projects/PointRend/point_rend/point_features.py b/projects/PointRend/point_rend/point_features.py -index 26e706e..9d1ab3b 100644 ---- a/projects/PointRend/point_rend/point_features.py -+++ b/projects/PointRend/point_rend/point_features.py -@@ -25,6 +25,88 @@ def _as_tensor(x): - return torch.as_tensor(x) - - -+def bilinear_grid_sample(im, grid, align_corners=False): -+ """Given an input and a flow-field grid, computes the output using input -+ values and pixel locations from grid. Supported only bilinear interpolation -+ method to sample the input pixels. -+ -+ Args: -+ im (torch.Tensor): Input feature map, shape (N, C, H, W) -+ grid (torch.Tensor): Point coordinates, shape (N, Hg, Wg, 2) -+ align_corners {bool}: If set to True, the extrema (-1 and 1) are -+ considered as referring to the center points of the input's -+ corner pixels. If set to False, they are instead considered as -+ referring to the corner points of the input's corner pixels, -+ making the sampling more resolution agnostic. -+ Returns: -+ torch.Tensor: A tensor with sampled points, shape (N, C, Hg, Wg) -+ """ -+ n, c, h, w = im.shape -+ gn, gh, gw, _ = grid.shape -+ assert n == gn -+ -+ x = grid[:, :, :, 0] -+ y = grid[:, :, :, 1] -+ -+ if align_corners: -+ x = ((x + 1) / 2) * (w - 1) -+ y = ((y + 1) / 2) * (h - 1) -+ else: -+ x = ((x + 1) * w - 1) / 2 -+ y = ((y + 1) * h - 1) / 2 -+ -+ x = x.view(n, -1) -+ y = y.view(n, -1) -+ -+ x0 = torch.floor(x).long() -+ y0 = torch.floor(y).long() -+ x1 = x0 + 1 -+ y1 = y0 + 1 -+ -+ wa = ((x1 - x) * (y1 - y)).unsqueeze(1) -+ wb = ((x1 - x) * (y - y0)).unsqueeze(1) -+ wc = ((x - x0) * (y1 - y)).unsqueeze(1) -+ wd = ((x - x0) * (y - y0)).unsqueeze(1) -+ -+ # Apply default for grid_sample function zero padding -+ im_padded = F.pad(im, pad=[1, 1, 1, 1], mode='constant', value=0) -+ padded_h = h + 2 -+ padded_w = w + 2 -+ # save points positions after padding -+ x0, x1, y0, y1 = x0 + 1, x1 + 1, y0 + 1, y1 + 1 -+ -+ # Clip coordinates to padded image size -+ x0 = torch.where(x0 < 0, torch.tensor(0), x0) -+ x0 = torch.where(x0 > padded_w - 1, torch.tensor(padded_w - 1), x0) -+ x1 = torch.where(x1 < 0, torch.tensor(0), x1) -+ x1 = torch.where(x1 > padded_w - 1, torch.tensor(padded_w - 1), x1) -+ y0 = torch.where(y0 < 0, torch.tensor(0), y0) -+ y0 = torch.where(y0 > padded_h - 1, torch.tensor(padded_h - 1), y0) -+ y1 = torch.where(y1 < 0, torch.tensor(0), y1) -+ y1 = torch.where(y1 > padded_h - 1, torch.tensor(padded_h - 1), y1) -+ -+ im_padded = im_padded.view(n, c, -1) -+ -+ x0_y0_tmp = (x0 + y0 * padded_w).unsqueeze(1).to(dtype=torch.int32) -+ s = torch.zeros(x0_y0_tmp.size(0), c, x0_y0_tmp.size(2)) -+ x0_y0 = x0_y0_tmp.expand_as(s).to(dtype=torch.int64) -+ x0_y1_tmp = (x0 + y1 * padded_w).unsqueeze(1).to(dtype=torch.int32) -+ s = torch.zeros(x0_y1_tmp.size(0), c, x0_y1_tmp.size(2)) -+ x0_y1 = x0_y1_tmp.expand_as(s).to(dtype=torch.int64) -+ x1_y0_tmp = (x1 + y0 * padded_w).unsqueeze(1).to(dtype=torch.int32) -+ s = torch.zeros(x1_y0_tmp.size(0), c, x1_y0_tmp.size(2)) -+ x1_y0 = x1_y0_tmp.expand_as(s).to(dtype=torch.int64) -+ x1_y1_tmp = (x1 + y1 * padded_w).unsqueeze(1).to(dtype=torch.int32) -+ s = torch.zeros(x1_y1_tmp.size(0), c, x1_y1_tmp.size(2)) -+ x1_y1 = x1_y1_tmp.expand_as(s).to(dtype=torch.int64) -+ -+ Ia = torch.gather(im_padded, 2, x0_y0) -+ Ib = torch.gather(im_padded, 2, x0_y1) -+ Ic = torch.gather(im_padded, 2, x1_y0) -+ Id = torch.gather(im_padded, 2, x1_y1) -+ -+ return (Ia * wa + Ib * wb + Ic * wc + Id * wd).reshape(n, c, gh, gw) -+ - def point_sample(input, point_coords, **kwargs): - """ - A wrapper around :function:`torch.nn.functional.grid_sample` to support 3D point_coords tensors. -@@ -44,8 +126,11 @@ def point_sample(input, point_coords, **kwargs): - add_dim = False - if point_coords.dim() == 3: - add_dim = True -- point_coords = point_coords.unsqueeze(2) -- output = F.grid_sample(input, 2.0 * point_coords - 1.0, **kwargs) -+ s = torch.zeros(point_coords.size(0), point_coords.size(1), 1, point_coords.size(2)) -+ point_coords = point_coords.reshape(point_coords.size(0), point_coords.size(1), 1, point_coords.size(2)) -+ #output = F.grid_sample(input, 2.0 * point_coords - 1.0, **kwargs) -+ # copy from mmcv -+ output = bilinear_grid_sample(input, 2.0 * point_coords - 1.0, align_corners=False) - if add_dim: - output = output.squeeze(3) - return output -@@ -147,8 +232,11 @@ def get_uncertain_point_coords_on_grid(uncertainty_map, num_points): - num_points = min(H * W, num_points) - point_indices = torch.topk(uncertainty_map.view(R, H * W), k=num_points, dim=1)[1] - point_coords = torch.zeros(R, num_points, 2, dtype=torch.float, device=uncertainty_map.device) -- point_coords[:, :, 0] = w_step / 2.0 + (point_indices % W).to(torch.float) * w_step -- point_coords[:, :, 1] = h_step / 2.0 + (point_indices // W).to(torch.float) * h_step -+ point_indices = point_indices.to(torch.int32) -+ x = w_step / 2.0 + (point_indices % W).to(torch.float) * w_step -+ y = h_step / 2.0 + (point_indices // W).to(torch.float) * h_step -+ point_coords = torch.stack([x, y], dim=-1) -+ point_indices = point_indices.to(torch.int64) - return point_indices, point_coords - - -diff --git a/projects/PointRend/point_rend/semantic_seg.py b/projects/PointRend/point_rend/semantic_seg.py -index ea65200..ba5d552 100644 ---- a/projects/PointRend/point_rend/semantic_seg.py -+++ b/projects/PointRend/point_rend/semantic_seg.py -@@ -126,7 +126,9 @@ class PointRendSemSegHead(nn.Module): - - # put sem seg point predictions to the right places on the upsampled grid. - N, C, H, W = sem_seg_logits.shape -- point_indices = point_indices.unsqueeze(1).expand(-1, C, -1) -+ point_indices = point_indices.unsqueeze(1) -+ s = torch.zeros(point_indices.size(0), C, point_indices.size(2)) -+ point_indices = point_indices.to(dtype=torch.int32).expand_as(s).to(dtype=torch.int64) - sem_seg_logits = ( - sem_seg_logits.reshape(N, C, H * W) - .scatter_(2, point_indices, point_logits) diff --git "a/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/ONNX\346\250\241\345\236\213\346\216\250\347\220\206\346\214\207\345\257\274/benchmark/cv/segmentation/PointRend_pth2onnx.py" "b/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/ONNX\346\250\241\345\236\213\346\216\250\347\220\206\346\214\207\345\257\274/benchmark/cv/segmentation/PointRend_pth2onnx.py" deleted file mode 100644 index f6588e243ebe31f590e53f7e75a0e70c095900e1..0000000000000000000000000000000000000000 --- "a/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/ONNX\346\250\241\345\236\213\346\216\250\347\220\206\346\214\207\345\257\274/benchmark/cv/segmentation/PointRend_pth2onnx.py" +++ /dev/null @@ -1,25 +0,0 @@ -import os -import torch -from detectron2.config import get_cfg -from detectron2.projects.point_rend import add_pointrend_config -from detectron2.engine import DefaultTrainer -from detectron2.checkpoint import DetectionCheckpointer - -def setup(): - cfg = get_cfg() - add_pointrend_config(cfg) - cfg.merge_from_file(os.getcwd() + '/projects/PointRend/configs/SemanticSegmentation/pointrend_semantic_R_101_FPN_1x_cityscapes.yaml') - cfg.MODEL.DEVICE = 'cpu' - cfg.freeze() - return cfg - -cfg = setup() -model = DefaultTrainer.build_model(cfg) -DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load('./model_final_cf6ac1.pkl') -model.eval() - -input_name = ['images'] -output_name = ['results'] -dynamic_axes = {'images': {0: '-1'}, 'results': {0: '-1'}} -dummy_input = torch.randn(1, 3, 1024, 2048) -torch.onnx.export(model, dummy_input, 'pointrend.onnx', input_names=input_name, dynamic_axes=dynamic_axes, output_names=output_name, opset_version=11, verbose=True) \ No newline at end of file diff --git "a/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/ONNX\346\250\241\345\236\213\346\216\250\347\220\206\346\214\207\345\257\274/benchmark/cv/segmentation/RefineDet.PyTorch" "b/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/ONNX\346\250\241\345\236\213\346\216\250\347\220\206\346\214\207\345\257\274/benchmark/cv/segmentation/RefineDet.PyTorch" deleted file mode 100644 index b8a24babb6ed49932c891499d7d8d47a4e9826bf..0000000000000000000000000000000000000000 --- "a/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/ONNX\346\250\241\345\236\213\346\216\250\347\220\206\346\214\207\345\257\274/benchmark/cv/segmentation/RefineDet.PyTorch" +++ /dev/null @@ -1 +0,0 @@ -直接导出onnx使用onnxsim后为空图 \ No newline at end of file diff --git "a/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/\344\270\223\351\242\230\346\241\210\344\276\213/\345\212\237\350\203\275\346\211\223\351\200\232/grid_sample\350\207\252\345\256\232\344\271\211onnx\347\256\227\345\255\220.md" "b/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/\344\270\223\351\242\230\346\241\210\344\276\213/\345\212\237\350\203\275\346\211\223\351\200\232/grid_sample\350\207\252\345\256\232\344\271\211onnx\347\256\227\345\255\220.md" new file mode 100644 index 0000000000000000000000000000000000000000..10b653903b8910320f0a0a2a2fc41658769030e5 --- /dev/null +++ "b/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/\344\270\223\351\242\230\346\241\210\344\276\213/\345\212\237\350\203\275\346\211\223\351\200\232/grid_sample\350\207\252\345\256\232\344\271\211onnx\347\256\227\345\255\220.md" @@ -0,0 +1,7 @@ +[PointRend](https://gitee.com/ascend/modelzoo/pulls/4611) +1.onnx不支持grid_sample算子,参考mmcv的自定义算子grid_sample的测试等价代码bilinear_grid_sample进行替换 +2.去除部分动态shape的where算子 +3.expand输入类型转换 +4.div输入类型转换 +5.padv3算子问题影响网络精度,通过修改代码进行了规避 +6.本模型是基于detectron2框架的模型 \ No newline at end of file diff --git "a/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/\344\270\223\351\242\230\346\241\210\344\276\213/\345\212\237\350\203\275\346\211\223\351\200\232/\345\257\274\345\207\272\347\232\204onnx\346\250\241\345\236\213\344\270\215\346\255\243\347\241\256\346\227\266\347\232\204\345\210\206\346\236\220.md" "b/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/\344\270\223\351\242\230\346\241\210\344\276\213/\345\212\237\350\203\275\346\211\223\351\200\232/\345\257\274\345\207\272\347\232\204onnx\346\250\241\345\236\213\344\270\215\346\255\243\347\241\256\346\227\266\347\232\204\345\210\206\346\236\220.md" new file mode 100644 index 0000000000000000000000000000000000000000..e5c3ed807fcbe07bc47249390d22098da9e6349e --- /dev/null +++ "b/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/\344\270\223\351\242\230\346\241\210\344\276\213/\345\212\237\350\203\275\346\211\223\351\200\232/\345\257\274\345\207\272\347\232\204onnx\346\250\241\345\236\213\344\270\215\346\255\243\347\241\256\346\227\266\347\232\204\345\210\206\346\236\220.md" @@ -0,0 +1,10 @@ +[refinedet](https://gitee.com/ascend/modelzoo/pulls/4007) +1.随机输入导出的onnx使用onnxsim后为空图,模型分段return导出onnx,最终发现随机输入时scores.size(0) == 0导致数据流为空,故导出onnx时使用真实图片作为输入 +2.nms含有动态shape小算子,替换为自定义batchnms暂未通,因nms在模型最后一部分故可放在后处理 +3.onnx转om时,不能使用fp16,否则精度不达标 + ``` + --precision_mode allow_fp32_to_fp16 + ``` +4.常量prior_data在后处理通过代码生成,不用经过模型计算 + +[仅供学习](https://gitee.com/wangjiangben_hw/ascend-pytorch-crowdintelligence-doc/tree/master/Ascend-PyTorch%E7%A6%BB%E7%BA%BF%E6%8E%A8%E7%90%86%E6%8C%87%E5%AF%BC/ONNX%E6%A8%A1%E5%9E%8B%E6%8E%A8%E7%90%86%E6%8C%87%E5%AF%BC/benchmark/cv/segmentation) \ No newline at end of file diff --git "a/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/\344\270\223\351\242\230\346\241\210\344\276\213/\345\212\237\350\203\275\346\211\223\351\200\232/\347\233\256\346\240\207\350\267\237\350\270\252SiamMask-\347\275\221\347\273\234\345\210\206\346\256\265.md" "b/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/\344\270\223\351\242\230\346\241\210\344\276\213/\345\212\237\350\203\275\346\211\223\351\200\232/\347\233\256\346\240\207\350\267\237\350\270\252SiamMask-\347\275\221\347\273\234\345\210\206\346\256\265.md" new file mode 100644 index 0000000000000000000000000000000000000000..89170d2fc9d10b1fcd324cc4976c504c69c100f0 --- /dev/null +++ "b/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/\344\270\223\351\242\230\346\241\210\344\276\213/\345\212\237\350\203\275\346\211\223\351\200\232/\347\233\256\346\240\207\350\267\237\350\270\252SiamMask-\347\275\221\347\273\234\345\210\206\346\256\265.md" @@ -0,0 +1,5 @@ +[SiamMask](https://gitee.com/ascend/modelzoo/pulls/4105) +1.om与onnx精度一致。 +2.因为SiamMask是前后帧连续处理,即上一帧的输入作为下一帧的输出,在Refine模块中进行动态pad,因此模型需要拆分为两段。因为SiamMask部分卷积存在使用自定义kernel来对输入进行卷积操作,导致卷积存在kernel和input的双输入的情况,[issue](http://github.com/onnx/onnx-tensorrt/issues/645),故在线推理测试性能。计算的是拆分的两部分模型合在一起完整推理的性能。 +3.因为SiamMask在corr部分固定了reshape之后的形状,并且针对前后帧连续处理,所以模型不支持多batch。 +4.推理速度较慢的问题目前出在io部分。如果`msame`支持将数据从内存中读写而不必须从`.bin`文件中读写,速度将会进一步加快。 \ No newline at end of file diff --git "a/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/\344\270\223\351\242\230\346\241\210\344\276\213/\345\212\237\350\203\275\346\211\223\351\200\232/\350\266\205\345\210\206\350\276\250\347\216\207\346\250\241\345\236\213\346\241\210\344\276\213.md" "b/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/\344\270\223\351\242\230\346\241\210\344\276\213/\345\212\237\350\203\275\346\211\223\351\200\232/\350\266\205\345\210\206\350\276\250\347\216\207\346\250\241\345\236\213\346\241\210\344\276\213.md" index 6836f045378caf74d5dbbfba63542263b779dc44..3a02a098247cbecc5a1f8bd66675519882518bbf 100644 --- "a/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/\344\270\223\351\242\230\346\241\210\344\276\213/\345\212\237\350\203\275\346\211\223\351\200\232/\350\266\205\345\210\206\350\276\250\347\216\207\346\250\241\345\236\213\346\241\210\344\276\213.md" +++ "b/Ascend-PyTorch\347\246\273\347\272\277\346\216\250\347\220\206\346\214\207\345\257\274/\344\270\223\351\242\230\346\241\210\344\276\213/\345\212\237\350\203\275\346\211\223\351\200\232/\350\266\205\345\210\206\350\276\250\347\216\207\346\250\241\345\236\213\346\241\210\344\276\213.md" @@ -1 +1,2 @@ -[SRFlow](https://gitee.com/ascend/modelzoo/pulls/3572) \ No newline at end of file +[SRFlow](https://gitee.com/ascend/modelzoo/pulls/3572) +[Cross-Scale-Non-Local-Attention](https://gitee.com/ascend/modelzoo/pulls/4395) \ No newline at end of file