From 856d470ed152b6a9c2196e6feb06f57b0c91443e Mon Sep 17 00:00:00 2001 From: underfitc Date: Wed, 4 Sep 2024 11:46:31 +0800 Subject: [PATCH 01/15] update dino --- .../MindIE-Torch/built-in/cv/DINO/README.md | 171 ++++++++++++++++++ .../built-in/cv/DINO/dino_compile.py | 131 ++++++++++++++ .../built-in/cv/DINO/model/dinov2_model.py | 56 ++++++ .../built-in/cv/DINO/onnx_export.py | 84 +++++++++ .../built-in/cv/DINO/perf_test_aie.py | 96 ++++++++++ .../built-in/cv/DINO/perf_test_onnx.py | 85 +++++++++ .../built-in/cv/DINO/precision_test.py | 160 ++++++++++++++++ 7 files changed, 783 insertions(+) create mode 100644 MindIE/MindIE-Torch/built-in/cv/DINO/README.md create mode 100644 MindIE/MindIE-Torch/built-in/cv/DINO/dino_compile.py create mode 100644 MindIE/MindIE-Torch/built-in/cv/DINO/model/dinov2_model.py create mode 100644 MindIE/MindIE-Torch/built-in/cv/DINO/onnx_export.py create mode 100644 MindIE/MindIE-Torch/built-in/cv/DINO/perf_test_aie.py create mode 100644 MindIE/MindIE-Torch/built-in/cv/DINO/perf_test_onnx.py create mode 100644 MindIE/MindIE-Torch/built-in/cv/DINO/precision_test.py diff --git a/MindIE/MindIE-Torch/built-in/cv/DINO/README.md b/MindIE/MindIE-Torch/built-in/cv/DINO/README.md new file mode 100644 index 0000000000..270e73f053 --- /dev/null +++ b/MindIE/MindIE-Torch/built-in/cv/DINO/README.md @@ -0,0 +1,171 @@ +# DINOv2-ViT-推理指导 + +- [概述](#ZH-CN_TOPIC_0000001172161123) + +- [推理环境准备](#ZH-CN_TOPIC_0000001126217823) + +- [快速上手](#ZH-CN_TOPIC_0000001126288456) + +- [模型推理性能精度](#ZH-CN_TOPIC_0000001172156835) + +# 概述 + +DINOv2是由Meta +AI开发的一个自监督学习方法，它专注于在无监督学习鲁棒的视觉特征。这个方法基于最初的DINO方法并进行了改进，提升了ViT模型在各种计算机视觉任务上的性能。([来自开源代码仓](https://github.com/facebookresearch/dinov2/tree/main)) + +# 推理环境准备\[所有版本\] + +- 该模型需要以下插件与驱动 + + **表 1** 版本配套表 + + | 配套 | 版本 | + |---------| ------- | + | 固件与驱动 | - | + | CANN | - | + | Python | 3.10.13 | + | PyTorch | 2.1.0 | + | MindIE | - | + + 注意：由于MindIE暂无支持该模型的商发版本，烦请用户联系华为工程师获取对应的固件驱动，CANN，MindIE PoC版本链接。 + 固件驱动和CANN的安装，请参考昇腾官方文档[环境快速部署](https://www.hiascend.com/document/detail/zh/quick-installation/24.0.RC1/quickinstg/800_3000/quickinstg_800_3000_0001.html)。 + + MindIE的安装需要先source toolkit的环境变量，然后直接安装，以默认安装路径`/usr/local/Ascend`为例： + ``` + source /usr/local/Ascend/ascend-tookit/set_env.sh + bash Ascend-mindie_*.run --install + ``` + +# 快速上手 + +1. 安装transformers + ```shell + pip install transformers==4.44.1 + ``` +2. 权重下载 + + | 模型 | 下载 | + |-------------|--------------------------------------------------------------------------------------------| + | ViT-S | [backbone only](https://huggingface.co/facebook/dinov2-small) | + | ViT-B | [backbone only](https://huggingface.co/facebook/dinov2-base) | + | ViT-L | [backbone only](https://huggingface.co/facebook/dinov2-large) | + | ViT-G | [backbone only](https://huggingface.co/facebook/dinov2-giant) | + + 按上述链接下载模型权重，以dinov2-vit-base为例 + ```shell + git lfs install + git clone https://huggingface.co/facebook/dinov2-base + ``` +2. 参数说明 + 导出和推理的参数命名有多数重合，公用说明如下： + + | 模型 | 下载 | + |---------------|---------------------------------------------| + | soc-version | 芯片类型，当前仅在Ascend910B4上调试 | + | device | NPU ID 号 | + | img-max-batch | 图片输入的最大batch-size | + | image-path | 输入图片地址 | + | model-version | 模型类型（"small", "base", "large", "giant"）| + | hf-model-path | 模型权重路径 | + | save-dir | 不同类型模型保存路径 | + 更多参数请参考运行不同脚本的`parse_args`部分 +3. ONNX模型导出 + ```shell + python onnx_export.py \ + --soc-version ${soc_version} \ + --image-path ${image_path} \ + --model-version ${model_version} \ + --hf-model-path ${hf_model_path} \ + --save-dir ${save_dir} + ``` + 执行完成后将在`save_dir`目录下生成`dinov2-${model_version}-onnx.pt`文件。 + +4. 模型编译 + ```shell + python dino_compile.py \ + --soc-version ${soc_version} \ + --device ${device} \ + --img-max-batch ${img_max_batch} \ + --image-path ${image_path} \ + --model-version ${model_version} \ + --model-path ${hf_model_path} \ + --save-dir ${save_dir} + ``` + 执行完成后将在`save_dir`目录下生成`dinov2-${model_version}-MindIETorch.pt`文件。 + +# 模型推理性能精度 + +1. 精度验证 + ```shell + dinov2_aie_path="./dinov2-${model_version}-MindIETorch.pt" + dinov2_onnx_path="./dinov2-${model_version}-onnx.pt" + python precision_test.py \ + --dinov2-aie-path ${dinov2_aie_path} \ + --dinov2-onnx-path ${dinov2_onnx_path} \ + --device ${device} \ + --image-path ${image_path} \ + --model-version ${model_version} \ + --hf-model-path ${hf_model_path} \ + ``` + 执行结束后，期望输出如下： + ``` + === Compare the outputs of ONNX and AIE dinov2 ${model_version} model === + Start comparing encoder... + Number of outputs to compare: 2 + Number of outputs with cosine similarity > 0.99: 2 + Number of outputs to compare: 2 + Number of outputs with cosine similarity > 0.99: 2 + ``` + +2. 性能验证 + + (a) aie模型性能测试 + ```shell + dinov2_aie_path="/home/hucong/old-DINO-V2/dinov2-${model_version}-MindIETorch.pt" + python perf_test_aie.py \ + --dinov2-aie-path ${dinov2_aie_path} \ + --device ${device} \ + --img-max-batch ${img_max_batch} \ + --hf-model-path ${hf_model_path} \ + ``` + + 执行结束后，期望输出如下（base）： + ``` + DINOV2 aie latency: 31.11 ms + DINOV2 aie throughput: 32.14 fps + ``` + + (b) onnx模型性能测试 + （可选）若使用GPU，请确保已安装CUDA和pytorch-gpu版本，同时需安装onnxruntime-gpu，如下所示： + ```shell + pip uninstall onnxruntime + pip install onnxruntime-gpu + ``` + 验证onnxruntime-gpu是否安装成功： + ```python + import onnxruntime + print(onnxruntime.get_device()) # 若输出为GPU，则说明安装成功 + ``` + 执行性能测试(CPU) + ```shell + dinov2_onnx_path="/home/hucong/old-DINO-V2/dinov2-${model_version}-onnx.pt" + python perf_test_onnx.py \ + --onnx-path ${dinov2_onnx_path} \ + --image-path ${image_path} \ + --hf-model-path ${hf_model_path} \ + ``` + + 执行结束后，期望输出如下（base）： + ``` + DINOV2 onnx latency: 268.65 ms + DINOV2 onnx throughput: 3.72 fps + ``` + + (c) 性能对比列表（GPU性能待测试）： + + | 模型 | MindIE-Torch（Ascend910B4） | ONNX（CPU） | + |---------|--------------------------------|---------------------| + | small | 24.60 ms / 40.64 fps | 193.84 ms / 5.16 fps | + | base | 31.11 ms / 32.14 fps | 268.65 ms / 3.72 fps | + | large | 120.52 ms / 8.30 fps | 533.02 ms / 1.88 fps | + | giant | 337.15 ms / 2.97 fps | 1121.83 ms / 0.89 fps | diff --git a/MindIE/MindIE-Torch/built-in/cv/DINO/dino_compile.py b/MindIE/MindIE-Torch/built-in/cv/DINO/dino_compile.py new file mode 100644 index 0000000000..b820afcb23 --- /dev/null +++ b/MindIE/MindIE-Torch/built-in/cv/DINO/dino_compile.py @@ -0,0 +1,131 @@ +# Copyright 2024 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import os +import time + +import mindietorch +import torch +from PIL import Image +from torch._export import export, dynamic_dim +from transformers import AutoImageProcessor + +from model.dinov2_model import Dinov2Model_WO_Embedding + + +def get_embed_input(args, model): + image = Image.open(args.image_path) + processor = AutoImageProcessor.from_pretrained(args.hf_model_path) + inputs = processor(images=Image.open(args.image_path), return_tensors="pt", padding=True) + embeddings = model.embeddings + embedding_output = embeddings(inputs.pixel_values) + return embedding_output + + +def export_dinov2(args): + model = Dinov2Model_WO_Embedding.from_pretrained(args.hf_model_path).float().eval() + embedding_output = get_embed_input(args, model) + embed_shape = embedding_output.shape + emb_input_shape = (args.img_max_batch, embed_shape[-2], embed_shape[-1]) + input_emb = torch.ones(emb_input_shape, dtype=torch.float32) + + constraints = [ + dynamic_dim(input_emb, 0) >= 1, + dynamic_dim(input_emb, 0) <= args.img_max_batch, + ] + + print("----- start exporting dynamic dinov2 -----") + intermediate_model = export( + model, + args=(input_emb,), + constraints=constraints + ) + print("----- export dynamic dinov2 success! -----") + return embed_shape, intermediate_model + + +def compile_dinov2(args): + # export dinov2 + embed_shape, intermediate_model = export_dinov2(args) + # compile dinov2 + mindietorch.set_device(args.device) + compile_inputs = [ + mindietorch.Input(min_shape=(1, embed_shape[-2], embed_shape[-1]), + max_shape=(args.img_max_batch, embed_shape[-2], embed_shape[-1])), + ] + + print("----- start mindietorch compile -----") + ts = time.time() + compiled_model = mindietorch.compile( + intermediate_model, + inputs=compile_inputs, + precision_policy=mindietorch._enums.PrecisionPolicy.FP16, + soc_version=args.soc_version, + ) + compile_cost = time.time() - ts + print(f"----- compile time cost: {compile_cost} -----") + print("----- end mindietorch compile -----") + + print("----- start saving -----") + model_save_dir = f"{args.save_dir}" + if not os.path.exists(model_save_dir): + os.makedirs(model_save_dir) + compiled_file_name = f"dinov2-{args.model_version}-MindIETorch.pt" + torch.save(compiled_model, model_save_dir + compiled_file_name, pickle_protocol=4) + print("----- saving done -----") + + +def parse_args(): + parser = argparse.ArgumentParser(description="Compile Dinov2-Vit model") + parser.add_argument( + "--soc-version", + default="Ascend910B4", + help="NPU version" + ) + parser.add_argument( + "--device", + type=int, + default=0 + ) + parser.add_argument( + "--img-max-batch", + type=int, + default=8 + ) + parser.add_argument( + "--image-path", + default="" + ) + parser.add_argument( + "--model-version", + default="base", + choices=["small", "base", "large", "giant"], + help="Specify the architecture of Dinov2-Vit model to be converted." + ) + parser.add_argument( + "--hf-model-path", + default="", + type=str, + help="Path of the Huggingface Dinov2-Vit model." + ) + parser.add_argument( + "--save-dir", + default="./" + ) + return parser.parse_args() + + +if __name__ == "__main__": + input_args = parse_args() + compile_dinov2(input_args) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINO/model/dinov2_model.py b/MindIE/MindIE-Torch/built-in/cv/DINO/model/dinov2_model.py new file mode 100644 index 0000000000..936fd38a2d --- /dev/null +++ b/MindIE/MindIE-Torch/built-in/cv/DINO/model/dinov2_model.py @@ -0,0 +1,56 @@ +# Copyright 2024 Huawei Technologies Co., Ltd +# Copyright 2024 the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional, Tuple + +import torch +from transformers.models.dinov2.modeling_dinov2 import Dinov2Model + + +class Dinov2Model_WO_Embedding(Dinov2Model): + def forward( + self, + embedding_output: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Tuple: + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if embedding_output is None: + raise ValueError("You have to specify embedding_output") + + head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers) + + encoder_outputs = self.encoder( + embedding_output, + head_mask=head_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = encoder_outputs[0] + sequence_output = self.layernorm(sequence_output) + pooled_output = sequence_output[:, 0, :] + + if not return_dict: + head_outputs = (sequence_output, pooled_output) + return head_outputs + encoder_outputs[1:] + + return sequence_output, pooled_output diff --git a/MindIE/MindIE-Torch/built-in/cv/DINO/onnx_export.py b/MindIE/MindIE-Torch/built-in/cv/DINO/onnx_export.py new file mode 100644 index 0000000000..ab59340584 --- /dev/null +++ b/MindIE/MindIE-Torch/built-in/cv/DINO/onnx_export.py @@ -0,0 +1,84 @@ +# Copyright 2024 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse + +import torch +import torch.onnx +from PIL import Image +from transformers import AutoImageProcessor, AutoModel + + +def convert_dinov2(args): + processor = AutoImageProcessor.from_pretrained(args.hf_model_path) + image = Image.open(args.image_path) + inputs = processor(images=(image,), return_tensors="pt") + pixel_values = inputs.pixel_values + model = AutoModel.from_pretrained(args.hf_model_path).float().eval() + + # 导出onnx模型 + onnx_path = f"{args.save_dir}dinov2-{args.model_version}-onnx.pt" + print("Starting to export dynamic onnx ...") + torch.onnx.export( + model, + (pixel_values,), + onnx_path, + input_names=["pixel_values"], + output_names=["sequence_output", "pooled_output"], + export_params=True, + opset_version=13, + verbose=True, + dynamic_axes={ + "pixel_values": {0: "image_batch_size"}, + "sequence_output": {0: "image_batch_size"}, + "pooled_output": {0: "image_batch_size"}, + } + ) + print("Successfully exported dynamic onnx!") + + +def parse_args(): + parser = argparse.ArgumentParser(description="Compile DINO model") + parser.add_argument( + "--soc-version", + default="Ascend910B4", + help="NPU version" + ) + parser.add_argument( + "--image-path", + type=str, + default="" + ) + parser.add_argument( + "--hf-model-path", + default="", + type=str, + help="Path of the Huggingface CLIP model." + ) + parser.add_argument( + "--model-version", + default="base", + choices=["small", "base", "large", "giant"], + help="Specify the architecture of Dinov2-Vit model to be converted." + ) + parser.add_argument( + "--save-dir", + default="./" + ) + args = parser.parse_args() + return args + + +if __name__ == "__main__": + args = parse_args() + convert_dinov2(args) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINO/perf_test_aie.py b/MindIE/MindIE-Torch/built-in/cv/DINO/perf_test_aie.py new file mode 100644 index 0000000000..d6422de3b6 --- /dev/null +++ b/MindIE/MindIE-Torch/built-in/cv/DINO/perf_test_aie.py @@ -0,0 +1,96 @@ +# Copyright 2024 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import json +import os +import time + +import mindietorch +import torch + +from model.dinov2_model import Dinov2Model_WO_Embedding + + +def test(inputs, model, stream, meta=""): + # warmup + for _ in range(10): + with mindietorch.npu.stream(stream): + model(*inputs) + # stream.synchronize() + + # performance test + num_infer = 100 + start = time.time() + for _ in range(num_infer): + with mindietorch.npu.stream(stream): + model(*inputs) + # stream.synchronize() + end = time.time() + + print(f"{meta} aie latency: {(end - start) / num_infer * 1000:.2f} ms") + print(f"{meta} aie throughput: {num_infer / (end - start):.2f} fps") + + +def dinov2_test(args): + device = f'npu:{args.device}' + stream = mindietorch.npu.Stream() + mindietorch.set_device(args.device) + + model = torch.load(args.dinov2_aie_path).eval().to(device) + hf_model = Dinov2Model_WO_Embedding.from_pretrained(args.hf_model_path).float().eval() + embeddings = hf_model.embeddings.to(device) + + hf_preprocessor_config_path = os.path.join(args.hf_model_path, "preprocessor_config.json") + if not os.path.exists(hf_preprocessor_config_path): + raise FileNotFoundError( + f"preprocessor_config.json not found at {args.hf_model_path}: {hf_preprocessor_config_path}") + with open(hf_preprocessor_config_path, "r") as f: + config_dict = json.load(f) + + image_height = config_dict["crop_size"]["height"] + image_width = config_dict["crop_size"]["width"] + img_input_shape = (args.img_max_batch, 3, image_height, image_width) + input_pixel_values = torch.randn(img_input_shape, dtype=torch.float32).to(device) + embedding_output = embeddings(input_pixel_values) + test([embedding_output], model, stream, "DINOV2") + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--dinov2-aie-path", + type=str, + default="" + ) + parser.add_argument( + "--device", + type=int, + help="NPU device id", + default=0 + ) + parser.add_argument( + "--hf-model-path", + default="" + ) + parser.add_argument( + "--img-max-batch", + type=int, + default=8 + ) + return parser.parse_args() + + +if __name__ == "__main__": + input_args = parse_args() + dinov2_test(input_args) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINO/perf_test_onnx.py b/MindIE/MindIE-Torch/built-in/cv/DINO/perf_test_onnx.py new file mode 100644 index 0000000000..ab4a87c4cd --- /dev/null +++ b/MindIE/MindIE-Torch/built-in/cv/DINO/perf_test_onnx.py @@ -0,0 +1,85 @@ +# Copyright 2024 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import time + +import onnxruntime as ort +from PIL import Image +from transformers import AutoImageProcessor + + +def test(encoder_path, provider, output_names, onnx_inputs, meta=""): + onnx_model = ort.InferenceSession( + encoder_path, + providers=[provider] + ) + + # warmup + for _ in range(10): + onnx_model.run(output_names, onnx_inputs) + # performance test + num_infer = 100 + start = time.time() + for _ in range(num_infer): + onnx_model.run(output_names, onnx_inputs) + end = time.time() + + print(f"{meta} onnx latency: {(end - start) / num_infer * 1000:.2f} ms") + print(f"{meta} onnx throughput: {num_infer / (end - start):.2f} fps") + + +def test_dinov2(args, provider): + image = Image.open(args.image_path) + processor = AutoImageProcessor.from_pretrained(args.hf_model_path) + + inputs = processor(images=(image,), return_tensors="pt") + pixel_values = inputs.pixel_values.detach().numpy() + + onnx_inputs = {"pixel_values": pixel_values} + output_names = ["sequence_output", "pooled_output"] + + test(args.onnx_path, provider, output_names, onnx_inputs, "DINOV2") + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--onnx-path", + type=str, + default="" + ) + parser.add_argument( + "--image-path", + type=str, + default="" + ) + parser.add_argument( + "--hf-model-path", + default="" + ) + parser.add_argument( + "--use-gpu", + action="store_true" + ) + return parser.parse_args() + + +if __name__ == "__main__": + input_args = parse_args() + if input_args.use_gpu: + provider = "CUDAExecutionProvider" + else: + provider = "CPUExecutionProvider" + + test_dinov2(input_args, provider) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINO/precision_test.py b/MindIE/MindIE-Torch/built-in/cv/DINO/precision_test.py new file mode 100644 index 0000000000..405bcff090 --- /dev/null +++ b/MindIE/MindIE-Torch/built-in/cv/DINO/precision_test.py @@ -0,0 +1,160 @@ +# Copyright 2024 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse + +import mindietorch +import numpy as np +import onnxruntime as ort +import torch +import torch.nn.functional as F +from PIL import Image +from transformers import AutoImageProcessor, AutoModel + +from model.dinov2_model import Dinov2Model_WO_Embedding + + +def compare_onnx_aie_output(onnx_out, aie_out, sim_threshold=0.99): + num_sim = 0 + for i, (a, b) in enumerate(zip(onnx_out, aie_out)): + a = a.reshape(1, -1).astype(np.float32) + b = b.reshape(1, -1) + sim = F.cosine_similarity(torch.from_numpy(a), b, dim=1) + if sim > sim_threshold: + num_sim += 1 + else: + print(f'Output {i} similarity: {sim}') + + print(f'Number of outputs to compare: {len(onnx_out)}') + print(f'Number of outputs with cosine similarity > {sim_threshold}: {num_sim}') + + +def compare_hf_aie_output(onnx_out, aie_out, sim_threshold=0.99): + num_sim = 0 + for i, (a, b) in enumerate(zip(onnx_out, aie_out)): + a = a.reshape(1, -1) + b = b.reshape(1, -1) + sim = F.cosine_similarity(a, b, dim=1) + if sim > sim_threshold: + num_sim += 1 + else: + print(f'Output {i} similarity: {sim}') + + print(f'Number of outputs to compare: {len(onnx_out)}') + print(f'Number of outputs with cosine similarity > {sim_threshold}: {num_sim}') + + +def get_embed_input(args): + device = f'npu:{args.device}' + # preprocess + processor = AutoImageProcessor.from_pretrained(args.hf_model_path) + inputs = processor(images=Image.open(args.image_path), return_tensors="pt", padding=True) + pixel_values = inputs.pixel_values.to(torch.float32).to(device) + model = Dinov2Model_WO_Embedding.from_pretrained(args.hf_model_path).float().eval().to(device) + embeddings = model.embeddings + embedding_output = embeddings(pixel_values) + return pixel_values, [embedding_output, ] + + +def compare(args): + device = f'npu:{args.device}' + pixel_values, embedding_output = get_embed_input(args) + # torch_npu + with torch.no_grad(): + hf_model = AutoModel.from_pretrained(args.hf_model_path).float().eval().to(device) + hf_outputs = hf_model(pixel_values) + hf_outputs = [hf_outputs.last_hidden_state, hf_outputs.pooler_output] + hf_out = [x.cpu().detach() for x in hf_outputs] + + # MindIETorch + mindietorch.set_device(args.device) + stream = mindietorch.npu.Stream(device) + aie_model = torch.load(args.dinov2_aie_path).to(device) + aie_model.eval() + + with mindietorch.npu.stream(stream): + aie_out = aie_model(*embedding_output) + if isinstance(aie_out, tuple) or isinstance(aie_out, list): + aie_out = [x.cpu() for x in aie_out] + else: + aie_out = aie_out.cpu() + + # ONNX + pixel_values = pixel_values.cpu().detach().numpy() + + if args.use_gpu: + provider = "CUDAExecutionProvider" + else: + provider = "CPUExecutionProvider" + + onnx_model = ort.InferenceSession( + args.dinov2_onnx_path, + providers=[provider] + ) + onnx_inputs = {"pixel_values": pixel_values} + output_names = ["sequence_output", "pooled_output"] + onnx_out = onnx_model.run(output_names, onnx_inputs) + + compare_onnx_aie_output(onnx_out, hf_out, args.sim_threshold) + compare_hf_aie_output(hf_out, aie_out, args.sim_threshold) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--dinov2-aie-path", + type=str, + default="" + ) + parser.add_argument( + "--dinov2-onnx-path", + type=str, + default="" + ) + parser.add_argument( + "--hf-model-path", + default="" + ) + parser.add_argument( + "--device", + type=int, + default=0, + help="NPU device id" + ) + parser.add_argument( + "--image-path", + type=str, + default="" + ) + parser.add_argument( + "--model-version", + default="base", + choices=["small", "base", "large", "giant"], + help="Specify the architecture of Dinov2-Vit model to be converted." + ) + parser.add_argument( + "--sim-threshold", + type=float, + default=0.99 + ) + parser.add_argument( + "--use-gpu", + action="store_true" + ) + return parser.parse_args() + + +if __name__ == "__main__": + input_args = parse_args() + print(f'=== Compare the outputs of ONNX and AIE dinov2 {input_args.model_version} model===') + compare(input_args) -- Gitee From 0f463f3176944191506bc114f0d7e61d5dedf49f Mon Sep 17 00:00:00 2001 From: underfitc Date: Wed, 4 Sep 2024 14:17:55 +0800 Subject: [PATCH 02/15] update --- MindIE/MindIE-Torch/built-in/cv/{DINO => DINOv2}/README.md | 0 MindIE/MindIE-Torch/built-in/cv/{DINO => DINOv2}/dino_compile.py | 0 .../built-in/cv/{DINO => DINOv2}/model/dinov2_model.py | 0 MindIE/MindIE-Torch/built-in/cv/{DINO => DINOv2}/onnx_export.py | 1 - .../MindIE-Torch/built-in/cv/{DINO => DINOv2}/perf_test_aie.py | 0 .../MindIE-Torch/built-in/cv/{DINO => DINOv2}/perf_test_onnx.py | 0 .../MindIE-Torch/built-in/cv/{DINO => DINOv2}/precision_test.py | 0 7 files changed, 1 deletion(-) rename MindIE/MindIE-Torch/built-in/cv/{DINO => DINOv2}/README.md (100%) rename MindIE/MindIE-Torch/built-in/cv/{DINO => DINOv2}/dino_compile.py (100%) rename MindIE/MindIE-Torch/built-in/cv/{DINO => DINOv2}/model/dinov2_model.py (100%) rename MindIE/MindIE-Torch/built-in/cv/{DINO => DINOv2}/onnx_export.py (99%) rename MindIE/MindIE-Torch/built-in/cv/{DINO => DINOv2}/perf_test_aie.py (100%) rename MindIE/MindIE-Torch/built-in/cv/{DINO => DINOv2}/perf_test_onnx.py (100%) rename MindIE/MindIE-Torch/built-in/cv/{DINO => DINOv2}/precision_test.py (100%) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINO/README.md b/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md similarity index 100% rename from MindIE/MindIE-Torch/built-in/cv/DINO/README.md rename to MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md diff --git a/MindIE/MindIE-Torch/built-in/cv/DINO/dino_compile.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/dino_compile.py similarity index 100% rename from MindIE/MindIE-Torch/built-in/cv/DINO/dino_compile.py rename to MindIE/MindIE-Torch/built-in/cv/DINOv2/dino_compile.py diff --git a/MindIE/MindIE-Torch/built-in/cv/DINO/model/dinov2_model.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/model/dinov2_model.py similarity index 100% rename from MindIE/MindIE-Torch/built-in/cv/DINO/model/dinov2_model.py rename to MindIE/MindIE-Torch/built-in/cv/DINOv2/model/dinov2_model.py diff --git a/MindIE/MindIE-Torch/built-in/cv/DINO/onnx_export.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/onnx_export.py similarity index 99% rename from MindIE/MindIE-Torch/built-in/cv/DINO/onnx_export.py rename to MindIE/MindIE-Torch/built-in/cv/DINOv2/onnx_export.py index ab59340584..777787fcab 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINO/onnx_export.py +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/onnx_export.py @@ -26,7 +26,6 @@ def convert_dinov2(args): pixel_values = inputs.pixel_values model = AutoModel.from_pretrained(args.hf_model_path).float().eval() - # 导出onnx模型 onnx_path = f"{args.save_dir}dinov2-{args.model_version}-onnx.pt" print("Starting to export dynamic onnx ...") torch.onnx.export( diff --git a/MindIE/MindIE-Torch/built-in/cv/DINO/perf_test_aie.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py similarity index 100% rename from MindIE/MindIE-Torch/built-in/cv/DINO/perf_test_aie.py rename to MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py diff --git a/MindIE/MindIE-Torch/built-in/cv/DINO/perf_test_onnx.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_onnx.py similarity index 100% rename from MindIE/MindIE-Torch/built-in/cv/DINO/perf_test_onnx.py rename to MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_onnx.py diff --git a/MindIE/MindIE-Torch/built-in/cv/DINO/precision_test.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/precision_test.py similarity index 100% rename from MindIE/MindIE-Torch/built-in/cv/DINO/precision_test.py rename to MindIE/MindIE-Torch/built-in/cv/DINOv2/precision_test.py -- Gitee From 9128b440472ab3dd0ae1f396c2258f9a705b8f41 Mon Sep 17 00:00:00 2001 From: underfitc Date: Wed, 4 Sep 2024 15:14:13 +0800 Subject: [PATCH 03/15] update --- MindIE/MindIE-Torch/built-in/cv/DINOv2/dino_compile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/dino_compile.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/dino_compile.py index b820afcb23..b944ce1d4e 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/dino_compile.py +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/dino_compile.py @@ -25,7 +25,6 @@ from model.dinov2_model import Dinov2Model_WO_Embedding def get_embed_input(args, model): - image = Image.open(args.image_path) processor = AutoImageProcessor.from_pretrained(args.hf_model_path) inputs = processor(images=Image.open(args.image_path), return_tensors="pt", padding=True) embeddings = model.embeddings -- Gitee From 1d29fa0455c292afbd43ff6ccaaf8ed31a33547e Mon Sep 17 00:00:00 2001 From: underfitc Date: Wed, 4 Sep 2024 15:34:21 +0800 Subject: [PATCH 04/15] update --- .../MindIE-Torch/built-in/cv/DINOv2/README.md | 4 +++- .../built-in/cv/DINOv2/onnx_export.py | 3 +-- .../built-in/cv/DINOv2/perf_test_aie.py | 18 +++++------------- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md b/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md index 270e73f053..ac22b032fe 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md @@ -38,9 +38,10 @@ AI开发的一个自监督学习方法，它专注于在无监督学习鲁棒的 # 快速上手 -1. 安装transformers +1. 安装依赖包 ```shell pip install transformers==4.44.1 + pip install numpy==1.26.4 ``` 2. 权重下载 @@ -81,6 +82,7 @@ AI开发的一个自监督学习方法，它专注于在无监督学习鲁棒的 执行完成后将在`save_dir`目录下生成`dinov2-${model_version}-onnx.pt`文件。 4. 模型编译 + 由于MindIETorch不支持mode为"bicubic"的nn.functional.interpolate，因此需要将模型中的embedding剥离出来进行在线推理，只编译模型encoder部分，执行以下脚本进行编译： ```shell python dino_compile.py \ --soc-version ${soc_version} \ diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/onnx_export.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/onnx_export.py index 777787fcab..c50abc54a4 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/onnx_export.py +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/onnx_export.py @@ -21,8 +21,7 @@ from transformers import AutoImageProcessor, AutoModel def convert_dinov2(args): processor = AutoImageProcessor.from_pretrained(args.hf_model_path) - image = Image.open(args.image_path) - inputs = processor(images=(image,), return_tensors="pt") + inputs = processor(images=Image.open(args.image_path), return_tensors="pt", padding=True) pixel_values = inputs.pixel_values model = AutoModel.from_pretrained(args.hf_model_path).float().eval() diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py index d6422de3b6..8df697d2dc 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py @@ -18,7 +18,8 @@ import time import mindietorch import torch - +from PIL import Image +from transformers import AutoImageProcessor from model.dinov2_model import Dinov2Model_WO_Embedding @@ -51,18 +52,9 @@ def dinov2_test(args): hf_model = Dinov2Model_WO_Embedding.from_pretrained(args.hf_model_path).float().eval() embeddings = hf_model.embeddings.to(device) - hf_preprocessor_config_path = os.path.join(args.hf_model_path, "preprocessor_config.json") - if not os.path.exists(hf_preprocessor_config_path): - raise FileNotFoundError( - f"preprocessor_config.json not found at {args.hf_model_path}: {hf_preprocessor_config_path}") - with open(hf_preprocessor_config_path, "r") as f: - config_dict = json.load(f) - - image_height = config_dict["crop_size"]["height"] - image_width = config_dict["crop_size"]["width"] - img_input_shape = (args.img_max_batch, 3, image_height, image_width) - input_pixel_values = torch.randn(img_input_shape, dtype=torch.float32).to(device) - embedding_output = embeddings(input_pixel_values) + processor = AutoImageProcessor.from_pretrained(args.hf_model_path) + inputs = processor(images=Image.open(args.image_path), return_tensors="pt", padding=True) + embedding_output = embeddings(inputs.input_pixel_values.to(device)) test([embedding_output], model, stream, "DINOV2") -- Gitee From 43a52d2f033e17e2cb6b4cc9f445e5ce134611af Mon Sep 17 00:00:00 2001 From: underfitc Date: Wed, 4 Sep 2024 16:08:04 +0800 Subject: [PATCH 05/15] update --- .../MindIE-Torch/built-in/cv/DINOv2/README.md | 14 +++++++------- .../built-in/cv/DINOv2/dino_compile.py | 6 +++--- .../built-in/cv/DINOv2/onnx_export.py | 17 ++++++++--------- .../built-in/cv/DINOv2/precision_test.py | 6 +++--- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md b/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md index ac22b032fe..ea81331096 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md @@ -46,7 +46,7 @@ AI开发的一个自监督学习方法，它专注于在无监督学习鲁棒的 2. 权重下载 | 模型 | 下载 | - |-------------|--------------------------------------------------------------------------------------------| + |-------------|--------------------------------------------------------------------------------------------| | ViT-S | [backbone only](https://huggingface.co/facebook/dinov2-small) | | ViT-B | [backbone only](https://huggingface.co/facebook/dinov2-base) | | ViT-L | [backbone only](https://huggingface.co/facebook/dinov2-large) | @@ -61,10 +61,10 @@ AI开发的一个自监督学习方法，它专注于在无监督学习鲁棒的 导出和推理的参数命名有多数重合，公用说明如下： | 模型 | 下载 | - |---------------|---------------------------------------------| + |---------------|---------------------------------------------| | soc-version | 芯片类型，当前仅在Ascend910B4上调试 | | device | NPU ID 号 | - | img-max-batch | 图片输入的最大batch-size | + | img-max-batch | 图片输入的最大batch size | | image-path | 输入图片地址 | | model-version | 模型类型（"small", "base", "large", "giant"）| | hf-model-path | 模型权重路径 | @@ -82,6 +82,7 @@ AI开发的一个自监督学习方法，它专注于在无监督学习鲁棒的 执行完成后将在`save_dir`目录下生成`dinov2-${model_version}-onnx.pt`文件。 4. 模型编译 + 由于MindIETorch不支持mode为"bicubic"的nn.functional.interpolate，因此需要将模型中的embedding剥离出来进行在线推理，只编译模型encoder部分，执行以下脚本进行编译： ```shell python dino_compile.py \ @@ -111,8 +112,7 @@ AI开发的一个自监督学习方法，它专注于在无监督学习鲁棒的 ``` 执行结束后，期望输出如下： ``` - === Compare the outputs of ONNX and AIE dinov2 ${model_version} model === - Start comparing encoder... + ----- Compare the outputs of ONNX and AIE dinov2 ${model_version} model ----- Number of outputs to compare: 2 Number of outputs with cosine similarity > 0.99: 2 Number of outputs to compare: 2 @@ -123,7 +123,7 @@ AI开发的一个自监督学习方法，它专注于在无监督学习鲁棒的 (a) aie模型性能测试 ```shell - dinov2_aie_path="/home/hucong/old-DINO-V2/dinov2-${model_version}-MindIETorch.pt" + dinov2_aie_path="./dinov2-${model_version}-MindIETorch.pt" python perf_test_aie.py \ --dinov2-aie-path ${dinov2_aie_path} \ --device ${device} \ @@ -150,7 +150,7 @@ AI开发的一个自监督学习方法，它专注于在无监督学习鲁棒的 ``` 执行性能测试(CPU) ```shell - dinov2_onnx_path="/home/hucong/old-DINO-V2/dinov2-${model_version}-onnx.pt" + dinov2_onnx_path="./dinov2-${model_version}-onnx.pt" python perf_test_onnx.py \ --onnx-path ${dinov2_onnx_path} \ --image-path ${image_path} \ diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/dino_compile.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/dino_compile.py index b944ce1d4e..b64c12a416 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/dino_compile.py +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/dino_compile.py @@ -86,7 +86,7 @@ def compile_dinov2(args): def parse_args(): - parser = argparse.ArgumentParser(description="Compile Dinov2-Vit model") + parser = argparse.ArgumentParser(description="Compile DINOv2-Vit model") parser.add_argument( "--soc-version", default="Ascend910B4", @@ -110,13 +110,13 @@ def parse_args(): "--model-version", default="base", choices=["small", "base", "large", "giant"], - help="Specify the architecture of Dinov2-Vit model to be converted." + help="Specify the architecture of DINOv2-Vit model to be converted." ) parser.add_argument( "--hf-model-path", default="", type=str, - help="Path of the Huggingface Dinov2-Vit model." + help="Path of the Huggingface DINOv2-Vit model." ) parser.add_argument( "--save-dir", diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/onnx_export.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/onnx_export.py index c50abc54a4..56e09e3d7d 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/onnx_export.py +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/onnx_export.py @@ -26,7 +26,7 @@ def convert_dinov2(args): model = AutoModel.from_pretrained(args.hf_model_path).float().eval() onnx_path = f"{args.save_dir}dinov2-{args.model_version}-onnx.pt" - print("Starting to export dynamic onnx ...") + print("----- Starting to export dynamic onnx -----") torch.onnx.export( model, (pixel_values,), @@ -42,11 +42,11 @@ def convert_dinov2(args): "pooled_output": {0: "image_batch_size"}, } ) - print("Successfully exported dynamic onnx!") + print("----- Successfully exported dynamic onnx! -----") def parse_args(): - parser = argparse.ArgumentParser(description="Compile DINO model") + parser = argparse.ArgumentParser(description="Compile DINOv2 model") parser.add_argument( "--soc-version", default="Ascend910B4", @@ -61,22 +61,21 @@ def parse_args(): "--hf-model-path", default="", type=str, - help="Path of the Huggingface CLIP model." + help="Path of the Huggingface DINOv2 model." ) parser.add_argument( "--model-version", default="base", choices=["small", "base", "large", "giant"], - help="Specify the architecture of Dinov2-Vit model to be converted." + help="Specify the architecture of DINOv2-Vit model to be converted." ) parser.add_argument( "--save-dir", default="./" ) - args = parser.parse_args() - return args + return parser.parse_args() if __name__ == "__main__": - args = parse_args() - convert_dinov2(args) + input_args = parse_args() + convert_dinov2(input_args) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/precision_test.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/precision_test.py index 405bcff090..7d859f26ea 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/precision_test.py +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/precision_test.py @@ -62,8 +62,8 @@ def get_embed_input(args): pixel_values = inputs.pixel_values.to(torch.float32).to(device) model = Dinov2Model_WO_Embedding.from_pretrained(args.hf_model_path).float().eval().to(device) embeddings = model.embeddings - embedding_output = embeddings(pixel_values) - return pixel_values, [embedding_output, ] + embedd_output = embeddings(pixel_values) + return pixel_values, [embedd_output, ] def compare(args): @@ -156,5 +156,5 @@ def parse_args(): if __name__ == "__main__": input_args = parse_args() - print(f'=== Compare the outputs of ONNX and AIE dinov2 {input_args.model_version} model===') + print(f'----- Compare the outputs of ONNX and AIE dinov2 {input_args.model_version} model -----') compare(input_args) -- Gitee From 3de037ef44f72ed07533839a6e052409da1059dc Mon Sep 17 00:00:00 2001 From: underfitc Date: Wed, 4 Sep 2024 16:27:05 +0800 Subject: [PATCH 06/15] update --- MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md | 1 + MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md b/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md index ea81331096..0e758a7027 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md @@ -127,6 +127,7 @@ AI开发的一个自监督学习方法，它专注于在无监督学习鲁棒的 python perf_test_aie.py \ --dinov2-aie-path ${dinov2_aie_path} \ --device ${device} \ + --image-path ${image_path} \ --img-max-batch ${img_max_batch} \ --hf-model-path ${hf_model_path} \ ``` diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py index 8df697d2dc..97cfac08c0 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py @@ -75,6 +75,11 @@ def parse_args(): "--hf-model-path", default="" ) + parser.add_argument( + "--image-path", + type=str, + default="" + ) parser.add_argument( "--img-max-batch", type=int, -- Gitee From 73efc98206c3e16831eee8a5c621f6f8806b2b49 Mon Sep 17 00:00:00 2001 From: underfitc Date: Wed, 4 Sep 2024 16:30:23 +0800 Subject: [PATCH 07/15] update --- MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py index 97cfac08c0..70cd11872f 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py @@ -54,7 +54,7 @@ def dinov2_test(args): processor = AutoImageProcessor.from_pretrained(args.hf_model_path) inputs = processor(images=Image.open(args.image_path), return_tensors="pt", padding=True) - embedding_output = embeddings(inputs.input_pixel_values.to(device)) + embedding_output = embeddings(inputs.pixel_values.to(device)) test([embedding_output], model, stream, "DINOV2") -- Gitee From b935ae7224ff58a559ef3542d18858f75d74232d Mon Sep 17 00:00:00 2001 From: underfitc Date: Wed, 4 Sep 2024 16:33:07 +0800 Subject: [PATCH 08/15] update --- MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py index 70cd11872f..60c5339129 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py @@ -28,7 +28,7 @@ def test(inputs, model, stream, meta=""): for _ in range(10): with mindietorch.npu.stream(stream): model(*inputs) - # stream.synchronize() + stream.synchronize() # performance test num_infer = 100 @@ -36,7 +36,7 @@ def test(inputs, model, stream, meta=""): for _ in range(num_infer): with mindietorch.npu.stream(stream): model(*inputs) - # stream.synchronize() + stream.synchronize() end = time.time() print(f"{meta} aie latency: {(end - start) / num_infer * 1000:.2f} ms") @@ -45,7 +45,7 @@ def test(inputs, model, stream, meta=""): def dinov2_test(args): device = f'npu:{args.device}' - stream = mindietorch.npu.Stream() + stream = mindietorch.npu.Stream(device) mindietorch.set_device(args.device) model = torch.load(args.dinov2_aie_path).eval().to(device) -- Gitee From c3b96702ea6e1dd583be839785364d41a936d955 Mon Sep 17 00:00:00 2001 From: underfitc Date: Wed, 4 Sep 2024 16:36:17 +0800 Subject: [PATCH 09/15] update --- MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py index 60c5339129..165bcb8e87 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py @@ -28,7 +28,6 @@ def test(inputs, model, stream, meta=""): for _ in range(10): with mindietorch.npu.stream(stream): model(*inputs) - stream.synchronize() # performance test num_infer = 100 @@ -36,7 +35,6 @@ def test(inputs, model, stream, meta=""): for _ in range(num_infer): with mindietorch.npu.stream(stream): model(*inputs) - stream.synchronize() end = time.time() print(f"{meta} aie latency: {(end - start) / num_infer * 1000:.2f} ms") -- Gitee From 3b7acda2423038e2d432403fcc240bccd66aa030 Mon Sep 17 00:00:00 2001 From: underfitc Date: Wed, 4 Sep 2024 16:43:53 +0800 Subject: [PATCH 10/15] update --- MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_onnx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_onnx.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_onnx.py index ab4a87c4cd..151e51f1f5 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_onnx.py +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_onnx.py @@ -19,9 +19,9 @@ from PIL import Image from transformers import AutoImageProcessor -def test(encoder_path, provider, output_names, onnx_inputs, meta=""): +def test(onnx_model_path, provider, output_names, onnx_inputs, meta=""): onnx_model = ort.InferenceSession( - encoder_path, + onnx_model_path, providers=[provider] ) -- Gitee From 964c6f45d1fcdb4e2525e07d87be24eb6d276911 Mon Sep 17 00:00:00 2001 From: underfitc Date: Wed, 4 Sep 2024 17:17:16 +0800 Subject: [PATCH 11/15] update --- MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md | 8 ++++---- MindIE/MindIE-Torch/built-in/cv/DINOv2/dino_compile.py | 2 +- MindIE/MindIE-Torch/built-in/cv/DINOv2/onnx_export.py | 2 +- MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py | 2 +- MindIE/MindIE-Torch/built-in/cv/DINOv2/precision_test.py | 6 +++--- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md b/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md index 0e758a7027..dc21c423af 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md @@ -168,7 +168,7 @@ AI开发的一个自监督学习方法，它专注于在无监督学习鲁棒的 | 模型 | MindIE-Torch（Ascend910B4） | ONNX（CPU） | |---------|--------------------------------|---------------------| - | small | 24.60 ms / 40.64 fps | 193.84 ms / 5.16 fps | - | base | 31.11 ms / 32.14 fps | 268.65 ms / 3.72 fps | - | large | 120.52 ms / 8.30 fps | 533.02 ms / 1.88 fps | - | giant | 337.15 ms / 2.97 fps | 1121.83 ms / 0.89 fps | + | small | 3.93 ms / 254.74 fps | 164.50 ms / 6.08 fps | + | base | 4.00 ms / 250.14 fps | 535.00 ms / 1.90 fps | + | large | 9.59 ms / 104.30 fps | 1086.95 ms / 0.92 fps | + | giant | 20.09 ms / 49.78 fps | 3088.63 ms / 0.32 fps | diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/dino_compile.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/dino_compile.py index b64c12a416..71654b0938 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/dino_compile.py +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/dino_compile.py @@ -26,7 +26,7 @@ from model.dinov2_model import Dinov2Model_WO_Embedding def get_embed_input(args, model): processor = AutoImageProcessor.from_pretrained(args.hf_model_path) - inputs = processor(images=Image.open(args.image_path), return_tensors="pt", padding=True) + inputs = processor(images=Image.open(args.image_path), return_tensors="pt") embeddings = model.embeddings embedding_output = embeddings(inputs.pixel_values) return embedding_output diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/onnx_export.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/onnx_export.py index 56e09e3d7d..d708219027 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/onnx_export.py +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/onnx_export.py @@ -21,7 +21,7 @@ from transformers import AutoImageProcessor, AutoModel def convert_dinov2(args): processor = AutoImageProcessor.from_pretrained(args.hf_model_path) - inputs = processor(images=Image.open(args.image_path), return_tensors="pt", padding=True) + inputs = processor(images=Image.open(args.image_path), return_tensors="pt") pixel_values = inputs.pixel_values model = AutoModel.from_pretrained(args.hf_model_path).float().eval() diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py index 165bcb8e87..18432a9577 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/perf_test_aie.py @@ -51,7 +51,7 @@ def dinov2_test(args): embeddings = hf_model.embeddings.to(device) processor = AutoImageProcessor.from_pretrained(args.hf_model_path) - inputs = processor(images=Image.open(args.image_path), return_tensors="pt", padding=True) + inputs = processor(images=Image.open(args.image_path), return_tensors="pt") embedding_output = embeddings(inputs.pixel_values.to(device)) test([embedding_output], model, stream, "DINOV2") diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/precision_test.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/precision_test.py index 7d859f26ea..eaa5aecf62 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/precision_test.py +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/precision_test.py @@ -58,12 +58,12 @@ def get_embed_input(args): device = f'npu:{args.device}' # preprocess processor = AutoImageProcessor.from_pretrained(args.hf_model_path) - inputs = processor(images=Image.open(args.image_path), return_tensors="pt", padding=True) + inputs = processor(images=Image.open(args.image_path), return_tensors="pt") pixel_values = inputs.pixel_values.to(torch.float32).to(device) model = Dinov2Model_WO_Embedding.from_pretrained(args.hf_model_path).float().eval().to(device) embeddings = model.embeddings - embedd_output = embeddings(pixel_values) - return pixel_values, [embedd_output, ] + embed_output = embeddings(pixel_values) + return pixel_values, [embed_output, ] def compare(args): -- Gitee From 1436a0300cdc47c825c1f2ca5da37cdba588f850 Mon Sep 17 00:00:00 2001 From: underfitc Date: Wed, 4 Sep 2024 17:19:19 +0800 Subject: [PATCH 12/15] update --- MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md b/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md index dc21c423af..61aaa03c0e 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md @@ -167,7 +167,7 @@ AI开发的一个自监督学习方法，它专注于在无监督学习鲁棒的 (c) 性能对比列表（GPU性能待测试）： | 模型 | MindIE-Torch（Ascend910B4） | ONNX（CPU） | - |---------|--------------------------------|---------------------| + |---------|--------------------------------|---------------------| | small | 3.93 ms / 254.74 fps | 164.50 ms / 6.08 fps | | base | 4.00 ms / 250.14 fps | 535.00 ms / 1.90 fps | | large | 9.59 ms / 104.30 fps | 1086.95 ms / 0.92 fps | -- Gitee From 3e853f4b515fde68b36ce566aa30a7c0c7941351 Mon Sep 17 00:00:00 2001 From: underfitc Date: Wed, 4 Sep 2024 17:39:10 +0800 Subject: [PATCH 13/15] update --- MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md b/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md index 61aaa03c0e..34d5f0e8de 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md @@ -165,10 +165,10 @@ AI开发的一个自监督学习方法，它专注于在无监督学习鲁棒的 ``` (c) 性能对比列表（GPU性能待测试）： - | 模型 | MindIE-Torch（Ascend910B4） | ONNX（CPU） | |---------|--------------------------------|---------------------| | small | 3.93 ms / 254.74 fps | 164.50 ms / 6.08 fps | | base | 4.00 ms / 250.14 fps | 535.00 ms / 1.90 fps | | large | 9.59 ms / 104.30 fps | 1086.95 ms / 0.92 fps | | giant | 20.09 ms / 49.78 fps | 3088.63 ms / 0.32 fps | + 不同机器的测试出的性能在绝对值上可能有一定差异（特别是CPU性能），但相对值差异是保持一致的。 \ No newline at end of file -- Gitee From 6d407b6ea35df27f28a498dedb77f8a915d19378 Mon Sep 17 00:00:00 2001 From: underfitc Date: Wed, 4 Sep 2024 17:55:18 +0800 Subject: [PATCH 14/15] update --- MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md | 2 ++ MindIE/MindIE-Torch/built-in/cv/DINOv2/model/dinov2_model.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md b/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md index 34d5f0e8de..eff50bbeac 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md @@ -80,6 +80,7 @@ AI开发的一个自监督学习方法，它专注于在无监督学习鲁棒的 --save-dir ${save_dir} ``` 执行完成后将在`save_dir`目录下生成`dinov2-${model_version}-onnx.pt`文件。 + 4. 模型编译 @@ -165,6 +166,7 @@ AI开发的一个自监督学习方法，它专注于在无监督学习鲁棒的 ``` (c) 性能对比列表（GPU性能待测试）： + | 模型 | MindIE-Torch（Ascend910B4） | ONNX（CPU） | |---------|--------------------------------|---------------------| | small | 3.93 ms / 254.74 fps | 164.50 ms / 6.08 fps | diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/model/dinov2_model.py b/MindIE/MindIE-Torch/built-in/cv/DINOv2/model/dinov2_model.py index 936fd38a2d..2021753184 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/model/dinov2_model.py +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/model/dinov2_model.py @@ -1,5 +1,5 @@ # Copyright 2024 Huawei Technologies Co., Ltd -# Copyright 2024 the HuggingFace Inc. team. All rights reserved. +# Copyright 2023 Meta AI and The HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -- Gitee From 52f2f01fa5249c7f952c7ea24ba5ba2c596aae5a Mon Sep 17 00:00:00 2001 From: underfitc Date: Wed, 4 Sep 2024 18:09:37 +0800 Subject: [PATCH 15/15] update --- MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md b/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md index eff50bbeac..cd10c15230 100644 --- a/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md +++ b/MindIE/MindIE-Torch/built-in/cv/DINOv2/README.md @@ -79,7 +79,7 @@ AI开发的一个自监督学习方法，它专注于在无监督学习鲁棒的 --hf-model-path ${hf_model_path} \ --save-dir ${save_dir} ``` - 执行完成后将在`save_dir`目录下生成`dinov2-${model_version}-onnx.pt`文件。 + 执行完成后将在`save_dir`目录下生成`dinov2-${model_version}-onnx.pt`文件。giant模型由于模型过大，导出时间较长，请耐心等待，并且会保存大量中间计算节点。 4. 模型编译 -- Gitee