diff --git a/ACL_PyTorch/contrib/cv/detection/YOLOF/LICENSE b/ACL_PyTorch/contrib/cv/detection/YOLOF/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..0561f7dca719dc2718f1fdc146feda1dea36a9ef --- /dev/null +++ b/ACL_PyTorch/contrib/cv/detection/YOLOF/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2021 Megvii, Base Detection + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/ACL_PyTorch/contrib/cv/detection/YOLOF/YOLOF.patch b/ACL_PyTorch/contrib/cv/detection/YOLOF/YOLOF.patch new file mode 100644 index 0000000000000000000000000000000000000000..a53f8fc6fef62bfe5fcbe222e650c50e67fc0816 --- /dev/null +++ b/ACL_PyTorch/contrib/cv/detection/YOLOF/YOLOF.patch @@ -0,0 +1,419 @@ +diff --git a/cvpods/evaluation/coco_evaluation.py b/cvpods/evaluation/coco_evaluation.py +index 2a4b4a0..36cb539 100644 +--- a/cvpods/evaluation/coco_evaluation.py ++++ b/cvpods/evaluation/coco_evaluation.py +@@ -16,7 +16,7 @@ from pycocotools.coco import COCO + import torch + + from cvpods.data.datasets.coco import convert_to_coco_json +-from cvpods.evaluation.fast_eval_api import COCOeval_opt as COCOeval ++from pycocotools.cocoeval import COCOeval + from cvpods.structures import Boxes, BoxMode, pairwise_iou + from cvpods.utils import PathManager, comm, create_small_table, create_table_with_header + +diff --git a/cvpods/evaluation/fast_eval_api.py b/cvpods/evaluation/fast_eval_api.py +index 55d0e26..9e0597e 100644 +--- a/cvpods/evaluation/fast_eval_api.py ++++ b/cvpods/evaluation/fast_eval_api.py +@@ -5,7 +5,6 @@ import time + import numpy as np + from pycocotools.cocoeval import COCOeval + +-from cvpods import _C + + + class COCOeval_opt(COCOeval): +diff --git a/cvpods/layers/border_align.py b/cvpods/layers/border_align.py +index 6fe47d0..e037514 100644 +--- a/cvpods/layers/border_align.py ++++ b/cvpods/layers/border_align.py +@@ -6,7 +6,6 @@ from torch import nn + from torch.autograd import Function + from torch.autograd.function import once_differentiable + +-from cvpods import _C + + + class BorderAlignFunc(Function): +diff --git a/cvpods/layers/deform_conv.py b/cvpods/layers/deform_conv.py +index 875b614..5b626c8 100644 +--- a/cvpods/layers/deform_conv.py ++++ b/cvpods/layers/deform_conv.py +@@ -8,7 +8,6 @@ from torch.autograd import Function + from torch.autograd.function import once_differentiable + from torch.nn.modules.utils import _pair + +-from cvpods import _C + + from .wrappers import _NewEmptyTensorOp + +diff --git a/cvpods/layers/nms.py b/cvpods/layers/nms.py +index 925886d..ec097df 100644 +--- a/cvpods/layers/nms.py ++++ b/cvpods/layers/nms.py +@@ -5,11 +5,10 @@ import torch + from torchvision.ops import boxes as box_ops + from torchvision.ops import nms # BC-compat + +-from cvpods import _C + from cvpods.layers.rotated_boxes import pairwise_iou_rotated + from cvpods.utils.apex_wrapper import float_function + +-ml_nms = _C.ml_nms ++ml_nms = None + + + @float_function +diff --git a/cvpods/layers/psroi_pool.py b/cvpods/layers/psroi_pool.py +index 3340db1..b0143e5 100644 +--- a/cvpods/layers/psroi_pool.py ++++ b/cvpods/layers/psroi_pool.py +@@ -6,7 +6,6 @@ import torch + from torch import nn + from torch.autograd import Function + +-from cvpods import _C + + + class _PSROIPool(Function): +diff --git a/cvpods/layers/roi_align.py b/cvpods/layers/roi_align.py +index fc087e4..4502cc3 100644 +--- a/cvpods/layers/roi_align.py ++++ b/cvpods/layers/roi_align.py +@@ -4,7 +4,6 @@ from torch.autograd import Function + from torch.autograd.function import once_differentiable + from torch.nn.modules.utils import _pair + +-from cvpods import _C + from cvpods.utils.apex_wrapper import float_function + + +diff --git a/cvpods/layers/roi_align_rotated.py b/cvpods/layers/roi_align_rotated.py +index 7e6bd52..c2a3997 100644 +--- a/cvpods/layers/roi_align_rotated.py ++++ b/cvpods/layers/roi_align_rotated.py +@@ -4,7 +4,6 @@ from torch.autograd import Function + from torch.autograd.function import once_differentiable + from torch.nn.modules.utils import _pair + +-from cvpods import _C + + + class _ROIAlignRotated(Function): +diff --git a/cvpods/layers/rotated_boxes.py b/cvpods/layers/rotated_boxes.py +index 3be7a11..326c264 100644 +--- a/cvpods/layers/rotated_boxes.py ++++ b/cvpods/layers/rotated_boxes.py +@@ -2,7 +2,6 @@ + from __future__ import absolute_import, division, print_function, unicode_literals + + # import torch +-from cvpods import _C + + + def pairwise_iou_rotated(boxes1, boxes2): +diff --git a/cvpods/layers/swap_align2nat.py b/cvpods/layers/swap_align2nat.py +index dd3798b..83eb6f7 100644 +--- a/cvpods/layers/swap_align2nat.py ++++ b/cvpods/layers/swap_align2nat.py +@@ -2,7 +2,6 @@ from torch import nn + from torch.autograd import Function + from torch.autograd.function import once_differentiable + +-from cvpods import _C + + + class _SwapAlign2Nat(Function): +diff --git a/cvpods/layers/tree_filter_core.py b/cvpods/layers/tree_filter_core.py +index ea8ec77..5e8c0f6 100644 +--- a/cvpods/layers/tree_filter_core.py ++++ b/cvpods/layers/tree_filter_core.py +@@ -7,7 +7,6 @@ from torch import nn + from torch.autograd import Function + from torch.autograd.function import once_differentiable + +-from cvpods import _C + + # pylint: disable=W0613 + +diff --git a/cvpods/modeling/anchor_generator.py b/cvpods/modeling/anchor_generator.py +index f57fffc..b69d059 100644 +--- a/cvpods/modeling/anchor_generator.py ++++ b/cvpods/modeling/anchor_generator.py +@@ -195,7 +195,7 @@ class DefaultAnchorGenerator(nn.Module): + boxes = Boxes(anchors_per_feature_map) + anchors_in_image.append(boxes) + +- anchors = [copy.deepcopy(anchors_in_image) for _ in range(num_images)] ++ anchors = anchors_in_image + return anchors + + +diff --git a/cvpods/modeling/box_regression.py b/cvpods/modeling/box_regression.py +index 56f2ccb..7225d3c 100644 +--- a/cvpods/modeling/box_regression.py ++++ b/cvpods/modeling/box_regression.py +@@ -99,10 +99,16 @@ class Box2BoxTransform(object): + ctr_y = boxes[..., 1] + 0.5 * heights + + wx, wy, ww, wh = self.weights +- dx = deltas[..., 0::4] / wx +- dy = deltas[..., 1::4] / wy +- dw = deltas[..., 2::4] / ww +- dh = deltas[..., 3::4] / wh ++ if torch.onnx.is_in_onnx_export(): ++ dx = deltas[..., 0:1:] ++ dy = deltas[..., 1:2:] ++ dw = deltas[..., 2:3:] ++ dh = deltas[..., 3:4:] ++ else: ++ dx = deltas[..., 0::4] / wx ++ dy = deltas[..., 1::4] / wy ++ dw = deltas[..., 2::4] / ww ++ dh = deltas[..., 3::4] / wh + + # Prevent sending too large values into torch.exp() + dx_width = dx * widths[..., None] +@@ -122,11 +128,17 @@ class Box2BoxTransform(object): + pred_w = torch.exp(dw) * widths[..., None] + pred_h = torch.exp(dh) * heights[..., None] + +- pred_boxes = torch.zeros_like(deltas) +- pred_boxes[..., 0::4] = pred_ctr_x - 0.5 * pred_w # x1 +- pred_boxes[..., 1::4] = pred_ctr_y - 0.5 * pred_h # y1 +- pred_boxes[..., 2::4] = pred_ctr_x + 0.5 * pred_w # x2 +- pred_boxes[..., 3::4] = pred_ctr_y + 0.5 * pred_h # y2 ++ if torch.onnx.is_in_onnx_export(): ++ pred_boxes = torch.cat([pred_ctr_x - 0.5 * pred_w, ++ pred_ctr_y - 0.5 * pred_h, ++ pred_ctr_x + 0.5 * pred_w, ++ pred_ctr_y + 0.5 * pred_h], dim=1) ++ else: ++ pred_boxes = torch.zeros_like(deltas) ++ pred_boxes[..., 0::4] = pred_ctr_x - 0.5 * pred_w # x1 ++ pred_boxes[..., 1::4] = pred_ctr_y - 0.5 * pred_h # y1 ++ pred_boxes[..., 2::4] = pred_ctr_x + 0.5 * pred_w # x2 ++ pred_boxes[..., 3::4] = pred_ctr_y + 0.5 * pred_h # y2 + return pred_boxes + + +diff --git a/cvpods/modeling/losses/sigmoid_focal_loss.py b/cvpods/modeling/losses/sigmoid_focal_loss.py +index 1fd5949..7cdc8ed 100644 +--- a/cvpods/modeling/losses/sigmoid_focal_loss.py ++++ b/cvpods/modeling/losses/sigmoid_focal_loss.py +@@ -7,7 +7,6 @@ from torch import nn + from torch.autograd import Function + from torch.autograd.function import once_differentiable + +-from cvpods import _C + + + # TODO: Use JIT to replace CUDA implementation in the future. +diff --git a/playground/detection/coco/yolof/yolof.cspdarknet53.DC5.9x/net.py b/playground/detection/coco/yolof/yolof.cspdarknet53.DC5.9x/net.py +index ab3aa15..48887f5 100644 +--- a/playground/detection/coco/yolof/yolof.cspdarknet53.DC5.9x/net.py ++++ b/playground/detection/coco/yolof/yolof.cspdarknet53.DC5.9x/net.py +@@ -5,7 +5,7 @@ from cvpods.layers import ShapeSpec + from cvpods.modeling.backbone import Backbone + from cvpods.modeling.anchor_generator import DefaultAnchorGenerator + +-from .cspdarknet import build_darknet_backbone ++from cspdarknet import build_darknet_backbone + sys.path.append("..") + from yolof_base import build_encoder, build_decoder, YOLOF + +diff --git a/playground/detection/coco/yolof/yolof_base/yolof.py b/playground/detection/coco/yolof/yolof_base/yolof.py +index c5143f9..ea86670 100644 +--- a/playground/detection/coco/yolof/yolof_base/yolof.py ++++ b/playground/detection/coco/yolof/yolof_base/yolof.py +@@ -3,6 +3,7 @@ from typing import List + + import torch + from torch import nn ++import torch.nn.functional as F + import torch.distributed as dist + + from cvpods.layers import ShapeSpec, cat, generalized_batched_nms +@@ -12,11 +13,58 @@ from cvpods.modeling.losses import sigmoid_focal_loss_jit + from cvpods.modeling.postprocessing import detector_postprocess + from cvpods.structures import Boxes, ImageList, Instances + from cvpods.utils import log_first_n ++from cvpods.modeling.anchor_generator import DefaultAnchorGenerator + + from .box_ops import box_iou, generalized_box_iou + from .uniform_matcher import UniformMatcher + + ++class BatchNMSOp(torch.autograd.Function): ++ @staticmethod ++ def forward(ctx, bboxes, scores, score_threshold, iou_threshold, max_size_per_class, max_total_size): ++ """ ++ boxes (torch.Tensor): boxes in shape (batch, N, C, 4). ++ scores (torch.Tensor): scores in shape (batch, N, C). ++ return: ++ nmsed_boxes: (batch, N, 4) ++ nmsed_scores: (batch, N) ++ nmsed_classes: (batch, N) ++ nmsed_num: (batch,) ++ """ ++ ++ # Phony implementation for onnx export ++ nmsed_boxes = bboxes[:, :max_total_size, 0, :] ++ nmsed_scores = scores[:, :max_total_size, 0] ++ nmsed_classes = torch.arange(max_total_size, dtype=torch.long) ++ nmsed_num = torch.Tensor([max_total_size]) ++ ++ return nmsed_boxes, nmsed_scores, nmsed_classes, nmsed_num ++ ++ @staticmethod ++ def symbolic(g, bboxes, scores, score_thr, iou_thr, max_size_p_class, max_t_size): ++ nmsed_boxes, nmsed_scores, nmsed_classes, nmsed_num = g.op('BatchMultiClassNMS', ++ bboxes, scores, score_threshold_f=score_thr, ++ iou_threshold_f=iou_thr, ++ max_size_per_class_i=max_size_p_class, ++ max_total_size_i=max_t_size, outputs=4) ++ return nmsed_boxes, nmsed_scores, nmsed_classes, nmsed_num ++ ++ ++def batch_nms_op(bboxes, scores, score_threshold, iou_threshold, class_num, max_size_per_class=100, max_total_size=100): ++ if bboxes.dtype == torch.float32: ++ bboxes = bboxes.half() ++ scores = scores.half() ++ bboxes = bboxes.reshape(-1, bboxes.shape[1].numpy(), 1, 4) ++ scores = scores.reshape(-1, scores.shape[1].numpy(), class_num) ++ nmsed_boxes, nmsed_scores, nmsed_classes, nmsed_num = BatchNMSOp.apply(bboxes, scores, ++ score_threshold, iou_threshold, ++ max_size_per_class, max_total_size) ++ nmsed_boxes = nmsed_boxes.float() ++ nmsed_scores = nmsed_scores.float() ++ nmsed_classes = nmsed_classes.long() ++ return nmsed_boxes, nmsed_scores, nmsed_classes ++ ++ + def permute_to_N_HWA_K(tensor, K): + """ + Transpose/reshape a tensor from (N, (A x K), H, W) to (N, (HxWxA), K) +@@ -53,18 +101,19 @@ class YOLOF(nn.Module): + self.nms_threshold = cfg.MODEL.YOLOF.NMS_THRESH_TEST + self.nms_type = cfg.MODEL.NMS_TYPE + self.max_detections_per_image = cfg.TEST.DETECTIONS_PER_IMAGE ++ self.cfg = cfg + # fmt: on + + self.backbone = cfg.build_backbone( + cfg, input_shape=ShapeSpec(channels=len(cfg.MODEL.PIXEL_MEAN))) + + backbone_shape = self.backbone.output_shape() +- feature_shapes = [backbone_shape[f] for f in self.in_features] ++ self.feature_shapes = [backbone_shape[f] for f in self.in_features] + self.encoder = cfg.build_encoder( + cfg, backbone_shape + ) + self.decoder = cfg.build_decoder(cfg) +- self.anchor_generator = cfg.build_anchor_generator(cfg, feature_shapes) ++ self.anchor_generator = cfg.build_anchor_generator(cfg, self.feature_shapes) + + # Matching and loss + self.box2box_transform = Box2BoxTransform( +@@ -83,6 +132,7 @@ class YOLOF(nn.Module): + torch.Tensor(cfg.MODEL.PIXEL_STD).to(self.device).view(3, 1, 1) + ) + self.to(self.device) ++ self._batch_size = 1 + + def forward(self, batched_inputs): + """ +@@ -145,6 +195,75 @@ class YOLOF(nn.Module): + processed_results.append({"instances": r}) + return processed_results + ++ def forward_onnx(self, inputs): ++ features = self.backbone(inputs) ++ features = [features[f] for f in self.in_features] ++ # return encode_tensor ++ box_cls, box_delta = self.decoder(self.encoder(features[0])) ++ anchors = [DefaultAnchorGenerator(self.cfg, self.feature_shapes)(features) for i in range(self._batch_size)] ++ results = self.inference_onnx([box_cls], [box_delta], anchors) ++ return results ++ ++ def inference_onnx(self, box_cls, box_delta, anchors): ++ boxes_all = [] ++ score_all = [] ++ ++ box_cls = [permute_to_N_HWA_K(x, self.num_classes) for x in box_cls] ++ box_delta = [permute_to_N_HWA_K(x, 4) for x in box_delta] ++ # list[Tensor], one per level, each has shape (N, Hi x Wi x A, K or 4) ++ ++ for img_idx, anchors_per_image in enumerate(anchors): ++ box_cls_per_image = [ ++ box_cls_per_level[img_idx] for box_cls_per_level in box_cls ++ ] ++ box_reg_per_image = [ ++ box_reg_per_level[img_idx] for box_reg_per_level in box_delta ++ ] ++ boxes, score = self.inference_single_image_onnx( ++ box_cls_per_image, box_reg_per_image, anchors_per_image) ++ boxes_all.append(boxes) ++ score_all.append(score) ++ boxes_all = torch.cat(boxes_all, dim=0) ++ score_all = torch.cat(score_all, dim=0) ++ nmsed_boxes, nmsed_scores, nmsed_classes = batch_nms_op(boxes_all, score_all, self.score_threshold, ++ self.nms_threshold, self.num_classes) ++ return nmsed_boxes, nmsed_scores, nmsed_classes ++ ++ def inference_single_image_onnx(self, box_cls, box_delta, anchors): ++ boxes_all = [] ++ scores_all = [] ++ class_idxs_all = [] ++ ++ # Iterate over every feature level ++ for box_cls_i, box_reg_i, anchors_i in zip(box_cls, box_delta, ++ anchors): ++ # (HxWxAxK,) ++ box_cls_i = box_cls_i.flatten().sigmoid_() ++ num_topk = min(self.topk_candidates, box_reg_i.size(0)) ++ predicted_prob, topk_idxs = torch.topk(box_cls_i, k=num_topk) ++ topk_idxs = topk_idxs.int() ++ anchor_idxs = topk_idxs // self.num_classes ++ classes_idxs = topk_idxs % self.num_classes ++ ++ anchor_idxs = anchor_idxs.long() ++ classes_idxs = classes_idxs.long() ++ ++ box_reg_i = box_reg_i[anchor_idxs] ++ anchors_i = anchors_i[anchor_idxs] ++ predicted_boxes = self.box2box_transform.apply_deltas( ++ box_reg_i, anchors_i.tensor) ++ boxes_all.append(predicted_boxes) ++ scores_all.append(predicted_prob) ++ class_idxs_all.append(classes_idxs) ++ boxes_all, scores_all, class_idxs_all = [ ++ cat(x) for x in [boxes_all, scores_all, class_idxs_all] ++ ] ++ scores_all = scores_all.reshape(-1, 1) ++ class_idxs_all = class_idxs_all.reshape(-1, 1) ++ scores_t = torch.zeros([scores_all.shape[0], self.num_classes], dtype=torch.float32) ++ scores_t = scores_t.scatter(1, class_idxs_all, scores_all) ++ return boxes_all.unsqueeze(0), scores_t.unsqueeze(0) ++ + def losses(self, + indices, + gt_instances, +diff --git a/setup.py b/setup.py +index 6280b8f..e5990f5 100644 +--- a/setup.py ++++ b/setup.py +@@ -74,13 +74,6 @@ def get_extensions(): + include_dirs = [extensions_dir] + + ext_modules = [ +- extension( +- "cvpods._C", +- sources, +- include_dirs=include_dirs, +- define_macros=define_macros, +- extra_compile_args=extra_compile_args, +- ) + ] + + return ext_modules diff --git a/ACL_PyTorch/contrib/cv/detection/YOLOF/YOLOF_postprocess.py b/ACL_PyTorch/contrib/cv/detection/YOLOF/YOLOF_postprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..1260f6dabfef837c403ba50e01a5de4c76dc5b37 --- /dev/null +++ b/ACL_PyTorch/contrib/cv/detection/YOLOF/YOLOF_postprocess.py @@ -0,0 +1,90 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np +import argparse + +import torch +from cvpods.structures import Boxes, Instances +from cvpods.modeling.postprocessing import detector_postprocess +from cvpods.engine import RUNNERS +from cvpods.evaluation import build_evaluator + +import sys +import os + +sys.path.append("{0}/YOLOF/playground/detection/coco/yolof/yolof.cspdarknet53.DC5.9x/".format(sys.path[0])) +from config import config + +const_shape = (608, 608) +dataset_name = "coco_2017_val" + + +def runner_decrator(cls): + def custom_build_evaluator(cls, cfg, dataset_name, dataset): + return build_evaluator(cfg, dataset_name, dataset, None, dump=True) + + cls.build_evaluator = classmethod(custom_build_evaluator) + return cls + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--pth_path', default="YOLOF_CSP_D_53_DC5_9x.pth") + parser.add_argument('--bin_data_path', default="result/") + parser.add_argument('--meta_info_path', default="yolof_meta.info") + parser.add_argument('--num_classes', default=81, type=int) + + args = parser.parse_args() + + opts = ['MODEL.WEIGHTS', args.pth_path, "MODEL.DEVICE", "cpu"] + config.merge_from_list(opts) + + cls = runner_decrator(RUNNERS.get(config.TRAINER.NAME)) + evaluator = cls.build_evaluator(config, dataset_name, cls.build_test_loader(config).dataset) + evaluator.reset() + bin_data_path = args.bin_data_path + os.listdir(args.bin_data_path)[0] + "/" + + with open(args.meta_info_path, "r") as fp: + for line in fp: + values = line.split() + file_name = values[0] + batch_size = (len(values) - 1) // 3 + nmsed_boxes_batch = np.fromfile("{0}{1}_output_{2}.bin".format(bin_data_path, file_name, 0), + dtype=np.float32).reshape(batch_size, -1, 4) + nmsed_scores_batch = np.fromfile("{0}{1}_output_{2}.bin".format(bin_data_path, file_name, 1), + dtype=np.float32).reshape(batch_size, -1) + nmsed_classes_batch = np.fromfile("{0}{1}_output_{2}.bin".format(bin_data_path, file_name, 2), + dtype=np.int64).reshape(batch_size, -1) + last_image = "" + for i in range(batch_size): + img_name = values[i * 3 + 1] + if img_name == last_image: + break + last_image = img_name + last_img_name = img_name + height = int(values[i * 3 + 2]) + width = int(values[i * 3 + 3]) + nmsed_boxes = nmsed_boxes_batch[i] + nmsed_scores = nmsed_scores_batch[i] + nmsed_classes = nmsed_classes_batch[i] + result = Instances(const_shape) + result.pred_boxes = Boxes(torch.tensor(nmsed_boxes)) + result.scores = torch.tensor(nmsed_scores) + result.pred_classes = torch.tensor(nmsed_classes) + r = detector_postprocess(result, height, width) + r = {"instances": r} + _input = {"image_id": int(img_name)} + evaluator.process([_input], [r]) + print(evaluator.evaluate()) + print(evaluator._dump_infos[0]['summary']) diff --git a/ACL_PyTorch/contrib/cv/detection/YOLOF/YOLOF_preprocess.py b/ACL_PyTorch/contrib/cv/detection/YOLOF/YOLOF_preprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..60344d400bb7528bfa8bd5ce0f80a8e8c18c1b73 --- /dev/null +++ b/ACL_PyTorch/contrib/cv/detection/YOLOF/YOLOF_preprocess.py @@ -0,0 +1,83 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import argparse +import numpy as np +from PIL import Image +import pickle as pk +import multiprocessing + +flags = None +width = 608 +height = 608 +pixel_mean = np.array([103.5300, 116.2800, 123.6750], dtype=np.float32) + + +def gen_input_bin(file_batches, batch): + i = 0 + image_bag = [] + image_meta_bag = [] + for file in file_batches[batch]: + i = i + 1 + print("batch", batch, file, "===", i) + src = Image.open(os.path.join(flags.image_src_path, file)).convert("RGB") + ori_shape = (src.size[1], src.size[0]) + image = src.resize((height, width), 2) + image = np.asarray(image) + image = image[..., ::-1] + image = image - pixel_mean + image = image.transpose(2, 0, 1) + image_meta = {'ori_shape': ori_shape, 'file_name': file.split('.')[0]} + image_bag.append(image) + image_meta_bag.append(image_meta) + if len(image_bag) % flags.batch_size == 0: + np.array(image_bag).tofile(os.path.join(flags.bin_file_path, "{}_{}.bin".format(batch, i))) + with open(os.path.join(flags.meta_file_path, "{}_{}.pk".format(batch, i)), "wb") as fp: + pk.dump(image_meta_bag, fp) + image_bag = [] + image_meta_bag = [] + if image_bag: + ext_img_bag = [image_bag[-1] for i in range(flags.batch_size-len(image_bag))] + image_bag += ext_img_bag + ext_img_meta_bag = [image_meta_bag[-1] for i in range(flags.batch_size - len(image_meta_bag))] + image_meta_bag += ext_img_meta_bag + np.array(image_bag).tofile(os.path.join(flags.bin_file_path, "{}_{}.bin".format(batch, i))) + with open(os.path.join(flags.meta_file_path, "{}_{}.pk".format(batch, i)), "wb") as fp: + pk.dump(image_meta_bag, fp) + + +def preprocess(): + step = 100 - (100 % flags.batch_size) + files = os.listdir(flags.image_src_path) + file_batches = [files[i:i + step] for i in range(0, 5000, step) if files[i:i + step] != []] + thread_pool = multiprocessing.Pool(len(file_batches)) + for batch in range(len(file_batches)): + thread_pool.apply_async(gen_input_bin, args=(file_batches, batch)) + thread_pool.close() + thread_pool.join() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='preprocess of YOLOF PyTorch model') + parser.add_argument("--image_src_path", default="YOLOF/datasets/coco/val2017", help='image of dataset') + parser.add_argument("--bin_file_path", default="val2017_bin") + parser.add_argument("--meta_file_path", default="val2017_bin_meta") + parser.add_argument("--batch_size", default=1, type=int) + flags = parser.parse_args() + if not os.path.exists(flags.bin_file_path): + os.makedirs(flags.bin_file_path) + if not os.path.exists(flags.meta_file_path): + os.makedirs(flags.meta_file_path) + preprocess() diff --git a/ACL_PyTorch/contrib/cv/detection/YOLOF/gen_dataset_info.py b/ACL_PyTorch/contrib/cv/detection/YOLOF/gen_dataset_info.py new file mode 100644 index 0000000000000000000000000000000000000000..de69f1132e110c24ada316149f2aa83f60f468be --- /dev/null +++ b/ACL_PyTorch/contrib/cv/detection/YOLOF/gen_dataset_info.py @@ -0,0 +1,36 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import argparse +import pickle as pk + +const_img_shape = (608, 608) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='generate dataset info file') + parser.add_argument("--meta_file_path", default="val2017_bin_meta") + parser.add_argument("--meta_info_file_name", default="yolof_meta.info") + args = parser.parse_args() + + with open(args.meta_info_file_name, "w") as fp1: + file_list = os.listdir(args.meta_file_path) + for file in file_list: + with open("{}/{}".format(args.meta_file_path, file), "rb") as fp2: + meta = pk.load(fp2) + fp1.write(file.split(".")[0]) + for dic in meta: + fp1.write(" {} {} {}".format(dic['file_name'], *dic['ori_shape'])) + fp1.write("\n") + print("Get info done!") diff --git a/ACL_PyTorch/contrib/cv/detection/YOLOF/modelzoo_level.txt b/ACL_PyTorch/contrib/cv/detection/YOLOF/modelzoo_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..7a41dd3e9f164a0414e50ad52a502c6d21b9b8ff --- /dev/null +++ b/ACL_PyTorch/contrib/cv/detection/YOLOF/modelzoo_level.txt @@ -0,0 +1,4 @@ +FuncStatus:OK +PrecisionStatus:OK +ModelConvert:OK +PerfStatus=OK diff --git a/ACL_PyTorch/contrib/cv/detection/YOLOF/pytorch2onnx.py b/ACL_PyTorch/contrib/cv/detection/YOLOF/pytorch2onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..4f111b8adf40ddc26875d3101cc6edceb097d981 --- /dev/null +++ b/ACL_PyTorch/contrib/cv/detection/YOLOF/pytorch2onnx.py @@ -0,0 +1,78 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import argparse +import numpy as np +from cvpods.engine import default_setup +from cvpods.checkpoint import DefaultCheckpointer + +import sys + +sys.path.append("{0}/YOLOF/playground/detection/coco/yolof/yolof.cspdarknet53.DC5.9x/".format(sys.path[0])) +sys.path.append("{0}/YOLOF/playground/detection/coco/yolof/".format(sys.path[0])) +from net import build_model +from config import config + +device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + + +def convert_batchnorm(module, process_group=None): + module_output = module + if isinstance(module, torch.nn.SyncBatchNorm): + module_output = torch.nn.BatchNorm2d(num_features=module.num_features, + eps=module.eps, + momentum=module.momentum, + affine=module.affine, + track_running_stats=module.track_running_stats) + if module.affine: + with torch.no_grad(): + module_output.weight = module.weight + module_output.bias = module.bias + module_output.running_mean = module.running_mean + module_output.running_var = module.running_var + module_output.num_batches_tracked = module.num_batches_tracked + for name, child in module.named_children(): + module_output.add_module(name, convert_batchnorm(child, process_group)) + del module + return module_output + + +def pth2onnx(args, fake_input, opts): + config.merge_from_list(opts) + model = build_model(config) + model._batch_size = args.batch_size + model.forward = model.forward_onnx + model = convert_batchnorm(model) + model.eval() + DefaultCheckpointer(model, save_dir=config.OUTPUT_DIR).resume_or_load( + config.MODEL.WEIGHTS, resume=False + ) + torch.onnx.export(model, fake_input, args.out, verbose=True, opset_version=11, + input_names=['input'], enable_onnx_checker=False) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--model_config', help='model config path', + default="YOLOF/playground/detection/coco/yolof/yolof.cspdarknet53.DC5.9x") + parser.add_argument('--out', help='onnx output name', default="yolof.onnx") + parser.add_argument('--pth_path', help='model pth path', default="./YOLOF_CSP_D_53_DC5_9x.pth") + parser.add_argument('--batch_size', type=int, default=1) + args = parser.parse_args() + + img_shape = (args.batch_size, 3, 608, 608) + fake_input = torch.randn(*img_shape) + opts = ['MODEL.WEIGHTS', args.pth_path, "MODEL.DEVICE", "cpu"] + pth2onnx(args, fake_input, opts) diff --git a/ACL_PyTorch/contrib/cv/detection/YOLOF/readme.md b/ACL_PyTorch/contrib/cv/detection/YOLOF/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..960872cef8254c43ebfd5b2a2906fa56c68c4d4f --- /dev/null +++ b/ACL_PyTorch/contrib/cv/detection/YOLOF/readme.md @@ -0,0 +1,47 @@ +### YOLOF模型PyTorch离线推理指导 + +### 1. 环境准备 + +1. 安装依赖 + +```bash +pip install -r requirements.txt +``` + +2. 获取，修改与安装开源模型代码 + +``` +git clone -b main https://github.com/megvii-model/YOLOF.git +cd YOLOF +git reset 6189487b80601dfeda89302c22abac060f977785 --hard + +patch -p1 < ../YOLOF.patch +python3 setup.py develop +cd .. +``` + +3. 将权重文件YOLOF_CSP_D_53_DC5_9x.pth放到当前工作目录 + +4. 数据集 + + 获取COCO数据集，并重命名为coco，放到当前目录下的 YOLOF/datasets/ 文件夹内 + +5. [获取msame工具](https://gitee.com/ascend/tools/tree/master/msame) + + 将msame文件放到当前工作目录 + +### 2. 离线推理 + +710上执行，执行时使npu-smi info查看设备状态，确保device空闲 + +```bash +bash test/pth2om.sh --batch_size=1 +bash test/eval_acc_perf.sh --batch_size=1 +``` + +**评测结果：** + +| 模型 | pth精度 | 710离线推理精度 | 710性能 | +| ---------- | ----------- | --------------- | ---------- | +| YOLOF bs1 | box AP:50.9 | box AP:51.0 | fps 27.697 | +| YOLOF bs16 | box AP:50.9 | box AP:51.0 | fps 38.069 | \ No newline at end of file diff --git a/ACL_PyTorch/contrib/cv/detection/YOLOF/requirements.txt b/ACL_PyTorch/contrib/cv/detection/YOLOF/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a0359b580427d4fb795d35597ebd680dd5871d56 --- /dev/null +++ b/ACL_PyTorch/contrib/cv/detection/YOLOF/requirements.txt @@ -0,0 +1,10 @@ +torch == 1.7.1 +torchvision == 0.8.2 + +onnx==1.7.0 +onnx-simplifier +opencv-python +cython +lvis +scipy==1.7.3 +sympy \ No newline at end of file diff --git a/ACL_PyTorch/contrib/cv/detection/YOLOF/test/eval_acc_perf.sh b/ACL_PyTorch/contrib/cv/detection/YOLOF/test/eval_acc_perf.sh new file mode 100644 index 0000000000000000000000000000000000000000..5c8f94f95267aa617c52ba4b14199ecedc384a7e --- /dev/null +++ b/ACL_PyTorch/contrib/cv/detection/YOLOF/test/eval_acc_perf.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +set -eu + +batch_size=1 + +for para in $* +do + if [[ $para == --batch_size* ]]; then + batch_size=`echo ${para#*=}` + fi +done + + +arch=`uname -m` + +rm -rf val2017_bin +rm -rf val2017_bin_meta +python YOLOF_preprocess.py \ +--bin_file_path val2017_bin \ +--meta_file_path val2017_bin_meta \ +--batch_size ${batch_size} + +if [ $? != 0 ]; then + echo "fail!" + exit -1 +fi + +python gen_dataset_info.py \ +--meta_file_path val2017_bin_meta \ +--meta_info_file_name yolof_meta.info + +if [ $? != 0 ]; then + echo "fail!" + exit -1 +fi + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +rm -rf result + +./msame --model "yolof.om" --input "val2017_bin" --output "result" --outfmt BIN + +if [ $? != 0 ]; then + echo "fail!" + exit -1 +fi + +python YOLOF_postprocess.py \ +--pth_path YOLOF_CSP_D_53_DC5_9x.pth \ +--bin_data_path result/ \ +--meta_info_path yolof_meta.info \ + +if [ $? != 0 ]; then + echo "fail!" + exit -1 +fi + diff --git a/ACL_PyTorch/contrib/cv/detection/YOLOF/test/parse.py b/ACL_PyTorch/contrib/cv/detection/YOLOF/test/parse.py new file mode 100644 index 0000000000000000000000000000000000000000..3fbf812ce48df315f7d4d9982c731333577c2a8a --- /dev/null +++ b/ACL_PyTorch/contrib/cv/detection/YOLOF/test/parse.py @@ -0,0 +1,32 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import json +import re + +if __name__ == '__main__': + if sys.argv[1].endswith('.json'): + result_json = sys.argv[1] + with open(result_json, 'r') as f: + content = f.read() + tops = [i.get('value') for i in json.loads(content).get('value') if 'Top' in i.get('key')] + print('om {} top1:{} top5:{}'.format(result_json.split('_')[1].split('.')[0], tops[0], tops[4])) + elif sys.argv[1].endswith('.txt'): + result_txt = sys.argv[1] + with open(result_txt, 'r') as f: + content = f.read() + txt_data_list = [i.strip() for i in re.findall(r':(.*?),', content.replace('\n', ',') + ',')] + fps = float(txt_data_list[7]) + print('710 bs{} fps:{}'.format(result_txt.split('_')[3], fps)) diff --git a/ACL_PyTorch/contrib/cv/detection/YOLOF/test/pth2om.sh b/ACL_PyTorch/contrib/cv/detection/YOLOF/test/pth2om.sh new file mode 100644 index 0000000000000000000000000000000000000000..3d89780778278e6566abd568033949484210fb3f --- /dev/null +++ b/ACL_PyTorch/contrib/cv/detection/YOLOF/test/pth2om.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -eu +batch_size=1 + +for para in $* +do + if [[ $para == --batch_size* ]]; then + batch_size=`echo ${para#*=}` + fi +done + + +rm -f yolof.onnx + +python pytorch2onnx.py \ +--model_config YOLOF/playground/detection/coco/yolof/yolof.cspdarknet53.DC5.9x \ +--out yolof.onnx \ +--pth_path ./YOLOF_CSP_D_53_DC5_9x.pth \ +--batch_size $batch_size + +if [ $? != 0 ]; then + echo "fail!" + exit -1 +fi + +rm -f yolof.om +source /usr/local/Ascend/ascend-toolkit/set_env.sh + +atc --framework=5 --model=yolof.onnx --output=yolof --input_format=NCHW \ +--input_shape="input:$batch_size,3,608,608" --log=error --soc_version=Ascend710 + +if [ -f "yolof.om" ]; then + echo "success" +else + echo "fail!" +fi