From 2ad2724ac83fbc115a84dccb236beb81b3b030f8 Mon Sep 17 00:00:00 2001 From: YoungPeng Date: Tue, 8 Oct 2024 14:49:07 +0800 Subject: [PATCH 01/10] Add: retinaface inference script. --- .../convnext_base/igie/README.md | 47 +++++ .../convnext_base/igie/build_engine.py | 73 +++++++ .../convnext_base/igie/export.py | 61 ++++++ .../convnext_base/igie/inference.py | 186 ++++++++++++++++++ .../infer_convnext_base_fp16_accuracy.sh | 35 ++++ .../infer_convnext_base_fp16_performance.sh | 36 ++++ 6 files changed, 438 insertions(+) create mode 100644 models/cv/classification/convnext_base/igie/README.md create mode 100644 models/cv/classification/convnext_base/igie/build_engine.py create mode 100644 models/cv/classification/convnext_base/igie/export.py create mode 100644 models/cv/classification/convnext_base/igie/inference.py create mode 100644 models/cv/classification/convnext_base/igie/scripts/infer_convnext_base_fp16_accuracy.sh create mode 100644 models/cv/classification/convnext_base/igie/scripts/infer_convnext_base_fp16_performance.sh diff --git a/models/cv/classification/convnext_base/igie/README.md b/models/cv/classification/convnext_base/igie/README.md new file mode 100644 index 00000000..442ad963 --- /dev/null +++ b/models/cv/classification/convnext_base/igie/README.md @@ -0,0 +1,47 @@ +# ConvNext Base + +## Description + +The ConvNeXt Base model represents a significant stride in the evolution of convolutional neural networks (CNNs), introduced by researchers at Facebook AI Research (FAIR) and UC Berkeley. It is part of the ConvNeXt family, which challenges the dominance of Vision Transformers (ViTs) in the realm of visual recognition tasks. + +## Setup + +### Install + +```bash +pip3 install onnx +pip3 install tqdm +``` + +### Download + +Pretrained model: + +Dataset: to download the validation dataset. + +### Model Conversion + +```bash +python3 export.py --weight convnext_base-6075fbad.pth --output convnext_base.onnx +``` + +## Inference + +```bash +export DATASETS_DIR=/Path/to/imagenet_val/ +``` + +### FP16 + +```bash +# Accuracy +bash scripts/infer_convnext_base_fp16_accuracy.sh +# Performance +bash scripts/infer_convnext_base_fp16_performance.sh +``` + +## Results + +| Model | BatchSize | Precision | FPS | Top-1(%) | Top-5(%) | +| -------------- | --------- | --------- | ------- | -------- | -------- | +| ConvNext Base | 32 | FP16 | 589.669 | 83.661 | 96.699 | diff --git a/models/cv/classification/convnext_base/igie/build_engine.py b/models/cv/classification/convnext_base/igie/build_engine.py new file mode 100644 index 00000000..d3626ae7 --- /dev/null +++ b/models/cv/classification/convnext_base/igie/build_engine.py @@ -0,0 +1,73 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import tvm +import argparse +from tvm import relay +from tvm.relay.import_model import import_model_to_igie + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--model_path", + type=str, + required=True, + help="original model path.") + + parser.add_argument("--engine_path", + type=str, + required=True, + help="igie export engine path.") + + parser.add_argument("--input", + type=str, + required=True, + help=""" + input info of the model, format should be: + input_name:input_shape + eg: --input input:1,3,224,224. + """) + + parser.add_argument("--precision", + type=str, + choices=["fp32", "fp16", "int8"], + required=True, + help="model inference precision.") + + args = parser.parse_args() + + return args + +def main(): + args = parse_args() + + # get input valueinfo + input_name, input_shape = args.input.split(":") + shape = tuple([int(s) for s in input_shape.split(",")]) + input_dict = {input_name: shape} + + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + + mod, params = import_model_to_igie(args.model_path, input_dict, backend="igie") + + # build engine + lib = tvm.relay.build(mod, target=target, params=params, precision=args.precision) + + # export engine + lib.export_library(args.engine_path) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/convnext_base/igie/export.py b/models/cv/classification/convnext_base/igie/export.py new file mode 100644 index 00000000..d9a2fe01 --- /dev/null +++ b/models/cv/classification/convnext_base/igie/export.py @@ -0,0 +1,61 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import torch +import torchvision +import argparse + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--weight", + type=str, + required=True, + help="pytorch model weight.") + + parser.add_argument("--output", + type=str, + required=True, + help="export onnx model path.") + + args = parser.parse_args() + return args + +def main(): + args = parse_args() + + model = torchvision.models.convnext_base() + model.load_state_dict(torch.load(args.weight)) + model.eval() + + input_names = ['input'] + output_names = ['output'] + dynamic_axes = {'input': {0: '-1'}, 'output': {0: '-1'}} + dummy_input = torch.randn(1, 3, 224, 224) + + torch.onnx.export( + model, + dummy_input, + args.output, + input_names = input_names, + dynamic_axes = dynamic_axes, + output_names = output_names, + opset_version=13 + ) + + print("Export onnx model successfully! ") + +if __name__ == "__main__": + main() diff --git a/models/cv/classification/convnext_base/igie/inference.py b/models/cv/classification/convnext_base/igie/inference.py new file mode 100644 index 00000000..3aef3ec7 --- /dev/null +++ b/models/cv/classification/convnext_base/igie/inference.py @@ -0,0 +1,186 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +import argparse +import tvm +import torch +import torchvision +import numpy as np +from tvm import relay +from tqdm import tqdm +from torchvision import transforms +from torchvision.transforms.functional import InterpolationMode + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--engine", + type=str, + required=True, + help="igie engine path.") + + parser.add_argument("--batchsize", + type=int, + required=True, + help="inference batch size.") + + parser.add_argument("--datasets", + type=str, + required=True, + help="datasets path.") + + parser.add_argument("--input_name", + type=str, + required=True, + help="input name of the model.") + + parser.add_argument("--warmup", + type=int, + default=3, + help="number of warmup before test.") + + parser.add_argument("--num_workers", + type=int, + default=16, + help="number of workers used in pytorch dataloader.") + + parser.add_argument("--acc_target", + type=float, + default=None, + help="Model inference Accuracy target.") + + parser.add_argument("--fps_target", + type=float, + default=None, + help="Model inference FPS target.") + + parser.add_argument("--perf_only", + type=bool, + default=False, + help="Run performance test only") + + args = parser.parse_args() + + return args + +def get_dataloader(data_path, batch_size, num_workers): + dataset = torchvision.datasets.ImageFolder( + data_path, + transforms.Compose( + [ + transforms.Resize(256, interpolation=InterpolationMode.BILINEAR), + transforms.CenterCrop(224), + transforms.PILToTensor(), + transforms.ConvertImageDtype(torch.float), + transforms.Normalize( + mean=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225) + ) + ] + ) + ) + + dataloader = torch.utils.data.DataLoader(dataset, batch_size, num_workers=num_workers) + + return dataloader + +def get_topk_accuracy(pred, label): + if isinstance(pred, np.ndarray): + pred = torch.from_numpy(pred) + + if isinstance(label, np.ndarray): + label = torch.from_numpy(label) + + top1_acc = 0 + top5_acc = 0 + for idx in range(len(label)): + label_value = label[idx] + if label_value == torch.topk(pred[idx].float(), 1).indices.data: + top1_acc += 1 + top5_acc += 1 + + elif label_value in torch.topk(pred[idx].float(), 5).indices.data: + top5_acc += 1 + + return top1_acc, top5_acc + +def main(): + args = parse_args() + + batch_size = args.batchsize + + # create iluvatar target & device + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + device = tvm.device(target.kind.name, 0) + + # load engine + lib = tvm.runtime.load_module(args.engine) + + # create runtime from engine + module = tvm.contrib.graph_executor.GraphModule(lib["default"](device)) + + # just run perf test + if args.perf_only: + ftimer = module.module.time_evaluator("run", device, number=100, repeat=1) + prof_res = np.array(ftimer().results) * 1000 + fps = batch_size * 1000 / np.mean(prof_res) + print(f"\n* Mean inference time: {np.mean(prof_res):.3f} ms, Mean fps: {fps:.3f}") + else: + # warm up + for _ in range(args.warmup): + module.run() + + # get dataloader + dataloader = get_dataloader(args.datasets, batch_size, args.num_workers) + + top1_acc = 0 + top5_acc = 0 + total_num = 0 + + for image, label in tqdm(dataloader): + + # pad the last batch + pad_batch = len(image) != batch_size + + if pad_batch: + origin_size = len(image) + image = np.resize(image, (batch_size, *image.shape[1:])) + + module.set_input(args.input_name, tvm.nd.array(image, device)) + + # run inference + module.run() + + pred = module.get_output(0).asnumpy() + + if pad_batch: + pred = pred[:origin_size] + + # get batch accuracy + batch_top1_acc, batch_top5_acc = get_topk_accuracy(pred, label) + + top1_acc += batch_top1_acc + top5_acc += batch_top5_acc + total_num += batch_size + + result_stat = {} + result_stat["acc@1"] = round(top1_acc / total_num * 100.0, 3) + result_stat["acc@5"] = round(top5_acc / total_num * 100.0, 3) + + print(f"\n* Top1 acc: {result_stat['acc@1']} %, Top5 acc: {result_stat['acc@5']} %") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/convnext_base/igie/scripts/infer_convnext_base_fp16_accuracy.sh b/models/cv/classification/convnext_base/igie/scripts/infer_convnext_base_fp16_accuracy.sh new file mode 100644 index 00000000..42575772 --- /dev/null +++ b/models/cv/classification/convnext_base/igie/scripts/infer_convnext_base_fp16_accuracy.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="convnext_base.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path convnext_base_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine convnext_base_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ No newline at end of file diff --git a/models/cv/classification/convnext_base/igie/scripts/infer_convnext_base_fp16_performance.sh b/models/cv/classification/convnext_base/igie/scripts/infer_convnext_base_fp16_performance.sh new file mode 100644 index 00000000..d5ae0649 --- /dev/null +++ b/models/cv/classification/convnext_base/igie/scripts/infer_convnext_base_fp16_performance.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="convnext_base.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path convnext_base_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine convnext_base_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ + --perf_only True \ No newline at end of file -- Gitee From 72c7d33b5a7055d53b3c15c1d7d9ba4e18bb0ecb Mon Sep 17 00:00:00 2001 From: YoungPeng Date: Tue, 8 Oct 2024 16:41:27 +0800 Subject: [PATCH 02/10] Add: densenet201 inference script. --- .../classification/densenet201/igie/README.md | 48 +++++ .../densenet201/igie/build_engine.py | 73 +++++++ .../classification/densenet201/igie/export.py | 74 +++++++ .../densenet201/igie/inference.py | 186 ++++++++++++++++++ .../infer_densenet201_fp16_accuracy.sh | 35 ++++ .../infer_densenet201_fp16_performance.sh | 36 ++++ 6 files changed, 452 insertions(+) create mode 100644 models/cv/classification/densenet201/igie/README.md create mode 100644 models/cv/classification/densenet201/igie/build_engine.py create mode 100644 models/cv/classification/densenet201/igie/export.py create mode 100644 models/cv/classification/densenet201/igie/inference.py create mode 100644 models/cv/classification/densenet201/igie/scripts/infer_densenet201_fp16_accuracy.sh create mode 100644 models/cv/classification/densenet201/igie/scripts/infer_densenet201_fp16_performance.sh diff --git a/models/cv/classification/densenet201/igie/README.md b/models/cv/classification/densenet201/igie/README.md new file mode 100644 index 00000000..595e338b --- /dev/null +++ b/models/cv/classification/densenet201/igie/README.md @@ -0,0 +1,48 @@ +# DenseNet201 + +## Description + +DenseNet201 is a deep convolutional neural network that stands out for its unique dense connection architecture, where each layer integrates features from all previous layers, effectively reusing features and reducing the number of parameters. This design not only enhances the network's information flow and parameter efficiency but also increases the model's regularization effect, helping to prevent overfitting. DenseNet201 consists of multiple dense blocks and transition layers, capable of capturing rich feature representations while maintaining computational efficiency, making it suitable for complex image recognition tasks. + + +## Setup + +### Install + +```bash +pip3 install onnx +pip3 install tqdm +``` + +### Download + +Pretrained model: + +Dataset: to download the validation dataset. + +### Model Conversion + +```bash +python3 export.py --weight densenet201-c1103571.pth --output densenet201.onnx +``` + +## Inference + +```bash +export DATASETS_DIR=/Path/to/imagenet_val/ +``` + +### FP16 + +```bash +# Accuracy +bash scripts/infer_densenet201_fp16_accuracy.sh +# Performance +bash scripts/infer_densenet201_fp16_performance.sh +``` + +## Results + +| Model | BatchSize | Precision | FPS | Top-1(%) | Top-5(%) | +| ----------- | --------- | --------- | -------- | -------- | -------- | +| DenseNet201 | 32 | FP16 | 758.592 | 76.851 | 93.338 | diff --git a/models/cv/classification/densenet201/igie/build_engine.py b/models/cv/classification/densenet201/igie/build_engine.py new file mode 100644 index 00000000..d3626ae7 --- /dev/null +++ b/models/cv/classification/densenet201/igie/build_engine.py @@ -0,0 +1,73 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import tvm +import argparse +from tvm import relay +from tvm.relay.import_model import import_model_to_igie + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--model_path", + type=str, + required=True, + help="original model path.") + + parser.add_argument("--engine_path", + type=str, + required=True, + help="igie export engine path.") + + parser.add_argument("--input", + type=str, + required=True, + help=""" + input info of the model, format should be: + input_name:input_shape + eg: --input input:1,3,224,224. + """) + + parser.add_argument("--precision", + type=str, + choices=["fp32", "fp16", "int8"], + required=True, + help="model inference precision.") + + args = parser.parse_args() + + return args + +def main(): + args = parse_args() + + # get input valueinfo + input_name, input_shape = args.input.split(":") + shape = tuple([int(s) for s in input_shape.split(",")]) + input_dict = {input_name: shape} + + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + + mod, params = import_model_to_igie(args.model_path, input_dict, backend="igie") + + # build engine + lib = tvm.relay.build(mod, target=target, params=params, precision=args.precision) + + # export engine + lib.export_library(args.engine_path) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/densenet201/igie/export.py b/models/cv/classification/densenet201/igie/export.py new file mode 100644 index 00000000..66019547 --- /dev/null +++ b/models/cv/classification/densenet201/igie/export.py @@ -0,0 +1,74 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import torch +import torchvision +import argparse +import re + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--weight", + type=str, + required=True, + help="pytorch model weight.") + + parser.add_argument("--output", + type=str, + required=True, + help="export onnx model path.") + + args = parser.parse_args() + return args + +def main(): + args = parse_args() + + model = torchvision.models.densenet201(weights=False) + + state_dict = torch.load(args.weight) + + pattern = re.compile(r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$' + ) + for key in list(state_dict.keys()): + res = pattern.match(key) + if res: + new_key = res.group(1) + res.group(2) + state_dict[new_key] = state_dict[key] + del state_dict[key] + + model.load_state_dict(state_dict) + model.eval() + + input_names = ['input'] + output_names = ['output'] + dynamic_axes = {'input': {0: '-1'}, 'output': {0: '-1'}} + dummy_input = torch.randn(1, 3, 224, 224) + + torch.onnx.export( + model, + dummy_input, + args.output, + input_names = input_names, + dynamic_axes = dynamic_axes, + output_names = output_names, + opset_version=13 + ) + + print("Export onnx model successfully! ") + +if __name__ == "__main__": + main() diff --git a/models/cv/classification/densenet201/igie/inference.py b/models/cv/classification/densenet201/igie/inference.py new file mode 100644 index 00000000..3aef3ec7 --- /dev/null +++ b/models/cv/classification/densenet201/igie/inference.py @@ -0,0 +1,186 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +import argparse +import tvm +import torch +import torchvision +import numpy as np +from tvm import relay +from tqdm import tqdm +from torchvision import transforms +from torchvision.transforms.functional import InterpolationMode + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--engine", + type=str, + required=True, + help="igie engine path.") + + parser.add_argument("--batchsize", + type=int, + required=True, + help="inference batch size.") + + parser.add_argument("--datasets", + type=str, + required=True, + help="datasets path.") + + parser.add_argument("--input_name", + type=str, + required=True, + help="input name of the model.") + + parser.add_argument("--warmup", + type=int, + default=3, + help="number of warmup before test.") + + parser.add_argument("--num_workers", + type=int, + default=16, + help="number of workers used in pytorch dataloader.") + + parser.add_argument("--acc_target", + type=float, + default=None, + help="Model inference Accuracy target.") + + parser.add_argument("--fps_target", + type=float, + default=None, + help="Model inference FPS target.") + + parser.add_argument("--perf_only", + type=bool, + default=False, + help="Run performance test only") + + args = parser.parse_args() + + return args + +def get_dataloader(data_path, batch_size, num_workers): + dataset = torchvision.datasets.ImageFolder( + data_path, + transforms.Compose( + [ + transforms.Resize(256, interpolation=InterpolationMode.BILINEAR), + transforms.CenterCrop(224), + transforms.PILToTensor(), + transforms.ConvertImageDtype(torch.float), + transforms.Normalize( + mean=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225) + ) + ] + ) + ) + + dataloader = torch.utils.data.DataLoader(dataset, batch_size, num_workers=num_workers) + + return dataloader + +def get_topk_accuracy(pred, label): + if isinstance(pred, np.ndarray): + pred = torch.from_numpy(pred) + + if isinstance(label, np.ndarray): + label = torch.from_numpy(label) + + top1_acc = 0 + top5_acc = 0 + for idx in range(len(label)): + label_value = label[idx] + if label_value == torch.topk(pred[idx].float(), 1).indices.data: + top1_acc += 1 + top5_acc += 1 + + elif label_value in torch.topk(pred[idx].float(), 5).indices.data: + top5_acc += 1 + + return top1_acc, top5_acc + +def main(): + args = parse_args() + + batch_size = args.batchsize + + # create iluvatar target & device + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + device = tvm.device(target.kind.name, 0) + + # load engine + lib = tvm.runtime.load_module(args.engine) + + # create runtime from engine + module = tvm.contrib.graph_executor.GraphModule(lib["default"](device)) + + # just run perf test + if args.perf_only: + ftimer = module.module.time_evaluator("run", device, number=100, repeat=1) + prof_res = np.array(ftimer().results) * 1000 + fps = batch_size * 1000 / np.mean(prof_res) + print(f"\n* Mean inference time: {np.mean(prof_res):.3f} ms, Mean fps: {fps:.3f}") + else: + # warm up + for _ in range(args.warmup): + module.run() + + # get dataloader + dataloader = get_dataloader(args.datasets, batch_size, args.num_workers) + + top1_acc = 0 + top5_acc = 0 + total_num = 0 + + for image, label in tqdm(dataloader): + + # pad the last batch + pad_batch = len(image) != batch_size + + if pad_batch: + origin_size = len(image) + image = np.resize(image, (batch_size, *image.shape[1:])) + + module.set_input(args.input_name, tvm.nd.array(image, device)) + + # run inference + module.run() + + pred = module.get_output(0).asnumpy() + + if pad_batch: + pred = pred[:origin_size] + + # get batch accuracy + batch_top1_acc, batch_top5_acc = get_topk_accuracy(pred, label) + + top1_acc += batch_top1_acc + top5_acc += batch_top5_acc + total_num += batch_size + + result_stat = {} + result_stat["acc@1"] = round(top1_acc / total_num * 100.0, 3) + result_stat["acc@5"] = round(top5_acc / total_num * 100.0, 3) + + print(f"\n* Top1 acc: {result_stat['acc@1']} %, Top5 acc: {result_stat['acc@5']} %") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/densenet201/igie/scripts/infer_densenet201_fp16_accuracy.sh b/models/cv/classification/densenet201/igie/scripts/infer_densenet201_fp16_accuracy.sh new file mode 100644 index 00000000..470b285a --- /dev/null +++ b/models/cv/classification/densenet201/igie/scripts/infer_densenet201_fp16_accuracy.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="densenet201.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path densenet201_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine densenet201_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ No newline at end of file diff --git a/models/cv/classification/densenet201/igie/scripts/infer_densenet201_fp16_performance.sh b/models/cv/classification/densenet201/igie/scripts/infer_densenet201_fp16_performance.sh new file mode 100644 index 00000000..e1ad69b7 --- /dev/null +++ b/models/cv/classification/densenet201/igie/scripts/infer_densenet201_fp16_performance.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="densenet201.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path densenet201_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine densenet201_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ + --perf_only True \ No newline at end of file -- Gitee From bcba9323816c4bff98238256ae9bcce6553f8ecf Mon Sep 17 00:00:00 2001 From: YoungPeng Date: Tue, 8 Oct 2024 17:09:37 +0800 Subject: [PATCH 03/10] Add: efficientnet_b3 inference script. --- .../efficientnet_b3/igie/README.md | 47 +++++ .../efficientnet_b3/igie/build_engine.py | 73 +++++++ .../efficientnet_b3/igie/export.py | 61 ++++++ .../efficientnet_b3/igie/inference.py | 186 ++++++++++++++++++ .../infer_efficientnet_b3_fp16_accuracy.sh | 35 ++++ .../infer_efficientnet_b3_fp16_performance.sh | 36 ++++ 6 files changed, 438 insertions(+) create mode 100644 models/cv/classification/efficientnet_b3/igie/README.md create mode 100644 models/cv/classification/efficientnet_b3/igie/build_engine.py create mode 100644 models/cv/classification/efficientnet_b3/igie/export.py create mode 100644 models/cv/classification/efficientnet_b3/igie/inference.py create mode 100644 models/cv/classification/efficientnet_b3/igie/scripts/infer_efficientnet_b3_fp16_accuracy.sh create mode 100644 models/cv/classification/efficientnet_b3/igie/scripts/infer_efficientnet_b3_fp16_performance.sh diff --git a/models/cv/classification/efficientnet_b3/igie/README.md b/models/cv/classification/efficientnet_b3/igie/README.md new file mode 100644 index 00000000..1331670b --- /dev/null +++ b/models/cv/classification/efficientnet_b3/igie/README.md @@ -0,0 +1,47 @@ +# EfficientNet B3 + +## Description + +EfficientNet B3 is a member of the EfficientNet family, a series of convolutional neural network architectures that are designed to achieve excellent accuracy and efficiency. Introduced by researchers at Google, EfficientNets utilize the compound scaling method, which uniformly scales the depth, width, and resolution of the network to improve accuracy and efficiency. + +## Setup + +### Install + +```bash +pip3 install onnx +pip3 install tqdm +``` + +### Download + +Pretrained model: + +Dataset: to download the validation dataset. + +### Model Conversion + +```bash +python3 export.py --weight efficientnet_b3_rwightman-b3899882.pth --output efficientnet_b3.onnx +``` + +## Inference + +```bash +export DATASETS_DIR=/Path/to/imagenet_val/ +``` + +### FP16 + +```bash +# Accuracy +bash scripts/infer_efficientnet_b3_fp16_accuracy.sh +# Performance +bash scripts/infer_efficientnet_b3_fp16_performance.sh +``` + +## Results + +| Model | BatchSize | Precision | FPS | Top-1(%) | Top-5(%) | +| --------------- | --------- | --------- | -------- | -------- | -------- | +| Efficientnet_b3 | 32 | FP16 | 1144.391 | 78.503 | 94.340 | diff --git a/models/cv/classification/efficientnet_b3/igie/build_engine.py b/models/cv/classification/efficientnet_b3/igie/build_engine.py new file mode 100644 index 00000000..d3626ae7 --- /dev/null +++ b/models/cv/classification/efficientnet_b3/igie/build_engine.py @@ -0,0 +1,73 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import tvm +import argparse +from tvm import relay +from tvm.relay.import_model import import_model_to_igie + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--model_path", + type=str, + required=True, + help="original model path.") + + parser.add_argument("--engine_path", + type=str, + required=True, + help="igie export engine path.") + + parser.add_argument("--input", + type=str, + required=True, + help=""" + input info of the model, format should be: + input_name:input_shape + eg: --input input:1,3,224,224. + """) + + parser.add_argument("--precision", + type=str, + choices=["fp32", "fp16", "int8"], + required=True, + help="model inference precision.") + + args = parser.parse_args() + + return args + +def main(): + args = parse_args() + + # get input valueinfo + input_name, input_shape = args.input.split(":") + shape = tuple([int(s) for s in input_shape.split(",")]) + input_dict = {input_name: shape} + + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + + mod, params = import_model_to_igie(args.model_path, input_dict, backend="igie") + + # build engine + lib = tvm.relay.build(mod, target=target, params=params, precision=args.precision) + + # export engine + lib.export_library(args.engine_path) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/efficientnet_b3/igie/export.py b/models/cv/classification/efficientnet_b3/igie/export.py new file mode 100644 index 00000000..f66e2cb0 --- /dev/null +++ b/models/cv/classification/efficientnet_b3/igie/export.py @@ -0,0 +1,61 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import torch +import torchvision +import argparse + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--weight", + type=str, + required=True, + help="pytorch model weight.") + + parser.add_argument("--output", + type=str, + required=True, + help="export onnx model path.") + + args = parser.parse_args() + return args + +def main(): + args = parse_args() + + model = torchvision.models.efficientnet_b3() + model.load_state_dict(torch.load(args.weight)) + model.eval() + + input_names = ['input'] + output_names = ['output'] + dynamic_axes = {'input': {0: '-1'}, 'output': {0: '-1'}} + dummy_input = torch.randn(1, 3, 224, 224) + + torch.onnx.export( + model, + dummy_input, + args.output, + input_names = input_names, + dynamic_axes = dynamic_axes, + output_names = output_names, + opset_version=13 + ) + + print("Export onnx model successfully! ") + +if __name__ == "__main__": + main() diff --git a/models/cv/classification/efficientnet_b3/igie/inference.py b/models/cv/classification/efficientnet_b3/igie/inference.py new file mode 100644 index 00000000..3aef3ec7 --- /dev/null +++ b/models/cv/classification/efficientnet_b3/igie/inference.py @@ -0,0 +1,186 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +import argparse +import tvm +import torch +import torchvision +import numpy as np +from tvm import relay +from tqdm import tqdm +from torchvision import transforms +from torchvision.transforms.functional import InterpolationMode + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--engine", + type=str, + required=True, + help="igie engine path.") + + parser.add_argument("--batchsize", + type=int, + required=True, + help="inference batch size.") + + parser.add_argument("--datasets", + type=str, + required=True, + help="datasets path.") + + parser.add_argument("--input_name", + type=str, + required=True, + help="input name of the model.") + + parser.add_argument("--warmup", + type=int, + default=3, + help="number of warmup before test.") + + parser.add_argument("--num_workers", + type=int, + default=16, + help="number of workers used in pytorch dataloader.") + + parser.add_argument("--acc_target", + type=float, + default=None, + help="Model inference Accuracy target.") + + parser.add_argument("--fps_target", + type=float, + default=None, + help="Model inference FPS target.") + + parser.add_argument("--perf_only", + type=bool, + default=False, + help="Run performance test only") + + args = parser.parse_args() + + return args + +def get_dataloader(data_path, batch_size, num_workers): + dataset = torchvision.datasets.ImageFolder( + data_path, + transforms.Compose( + [ + transforms.Resize(256, interpolation=InterpolationMode.BILINEAR), + transforms.CenterCrop(224), + transforms.PILToTensor(), + transforms.ConvertImageDtype(torch.float), + transforms.Normalize( + mean=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225) + ) + ] + ) + ) + + dataloader = torch.utils.data.DataLoader(dataset, batch_size, num_workers=num_workers) + + return dataloader + +def get_topk_accuracy(pred, label): + if isinstance(pred, np.ndarray): + pred = torch.from_numpy(pred) + + if isinstance(label, np.ndarray): + label = torch.from_numpy(label) + + top1_acc = 0 + top5_acc = 0 + for idx in range(len(label)): + label_value = label[idx] + if label_value == torch.topk(pred[idx].float(), 1).indices.data: + top1_acc += 1 + top5_acc += 1 + + elif label_value in torch.topk(pred[idx].float(), 5).indices.data: + top5_acc += 1 + + return top1_acc, top5_acc + +def main(): + args = parse_args() + + batch_size = args.batchsize + + # create iluvatar target & device + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + device = tvm.device(target.kind.name, 0) + + # load engine + lib = tvm.runtime.load_module(args.engine) + + # create runtime from engine + module = tvm.contrib.graph_executor.GraphModule(lib["default"](device)) + + # just run perf test + if args.perf_only: + ftimer = module.module.time_evaluator("run", device, number=100, repeat=1) + prof_res = np.array(ftimer().results) * 1000 + fps = batch_size * 1000 / np.mean(prof_res) + print(f"\n* Mean inference time: {np.mean(prof_res):.3f} ms, Mean fps: {fps:.3f}") + else: + # warm up + for _ in range(args.warmup): + module.run() + + # get dataloader + dataloader = get_dataloader(args.datasets, batch_size, args.num_workers) + + top1_acc = 0 + top5_acc = 0 + total_num = 0 + + for image, label in tqdm(dataloader): + + # pad the last batch + pad_batch = len(image) != batch_size + + if pad_batch: + origin_size = len(image) + image = np.resize(image, (batch_size, *image.shape[1:])) + + module.set_input(args.input_name, tvm.nd.array(image, device)) + + # run inference + module.run() + + pred = module.get_output(0).asnumpy() + + if pad_batch: + pred = pred[:origin_size] + + # get batch accuracy + batch_top1_acc, batch_top5_acc = get_topk_accuracy(pred, label) + + top1_acc += batch_top1_acc + top5_acc += batch_top5_acc + total_num += batch_size + + result_stat = {} + result_stat["acc@1"] = round(top1_acc / total_num * 100.0, 3) + result_stat["acc@5"] = round(top5_acc / total_num * 100.0, 3) + + print(f"\n* Top1 acc: {result_stat['acc@1']} %, Top5 acc: {result_stat['acc@5']} %") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/efficientnet_b3/igie/scripts/infer_efficientnet_b3_fp16_accuracy.sh b/models/cv/classification/efficientnet_b3/igie/scripts/infer_efficientnet_b3_fp16_accuracy.sh new file mode 100644 index 00000000..36e9a874 --- /dev/null +++ b/models/cv/classification/efficientnet_b3/igie/scripts/infer_efficientnet_b3_fp16_accuracy.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="efficientnet_b3.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path efficientnet_b3_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine efficientnet_b3_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ No newline at end of file diff --git a/models/cv/classification/efficientnet_b3/igie/scripts/infer_efficientnet_b3_fp16_performance.sh b/models/cv/classification/efficientnet_b3/igie/scripts/infer_efficientnet_b3_fp16_performance.sh new file mode 100644 index 00000000..27b13c90 --- /dev/null +++ b/models/cv/classification/efficientnet_b3/igie/scripts/infer_efficientnet_b3_fp16_performance.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="efficientnet_b3.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path efficientnet_b3_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine efficientnet_b3_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ + --perf_only True \ No newline at end of file -- Gitee From 589265d702113135f93d2c64073f7b4dd654e8db Mon Sep 17 00:00:00 2001 From: YoungPeng Date: Wed, 9 Oct 2024 11:07:11 +0800 Subject: [PATCH 04/10] Add: efficientnet_v2_s inference script. --- .../efficientnet_v2_s/igie/README.md | 47 +++++ .../efficientnet_v2_s/igie/build_engine.py | 73 +++++++ .../efficientnet_v2_s/igie/export.py | 61 ++++++ .../efficientnet_v2_s/igie/inference.py | 186 ++++++++++++++++++ .../infer_efficientnet_v2_s_fp16_accuracy.sh | 35 ++++ ...nfer_efficientnet_v2_s_fp16_performance.sh | 36 ++++ 6 files changed, 438 insertions(+) create mode 100644 models/cv/classification/efficientnet_v2_s/igie/README.md create mode 100644 models/cv/classification/efficientnet_v2_s/igie/build_engine.py create mode 100644 models/cv/classification/efficientnet_v2_s/igie/export.py create mode 100644 models/cv/classification/efficientnet_v2_s/igie/inference.py create mode 100644 models/cv/classification/efficientnet_v2_s/igie/scripts/infer_efficientnet_v2_s_fp16_accuracy.sh create mode 100644 models/cv/classification/efficientnet_v2_s/igie/scripts/infer_efficientnet_v2_s_fp16_performance.sh diff --git a/models/cv/classification/efficientnet_v2_s/igie/README.md b/models/cv/classification/efficientnet_v2_s/igie/README.md new file mode 100644 index 00000000..cd4fa6f8 --- /dev/null +++ b/models/cv/classification/efficientnet_v2_s/igie/README.md @@ -0,0 +1,47 @@ +# EfficientNet_v2_s + +## Description + +EfficientNetV2 S is an optimized model in the EfficientNetV2 series, which was developed by Google researchers. It continues the legacy of the EfficientNet family, focusing on advancing the state-of-the-art in accuracy and efficiency through advanced scaling techniques and architectural innovations. + +## Setup + +### Install + +```bash +pip3 install onnx +pip3 install tqdm +``` + +### Download + +Pretrained model: + +Dataset: to download the validation dataset. + +### Model Conversion + +```bash +python3 export.py --weight efficientnet_v2_s-dd5fe13b.pth --output efficientnet_v2_s.onnx +``` + +## Inference + +```bash +export DATASETS_DIR=/Path/to/imagenet_val/ +``` + +### FP16 + +```bash +# Accuracy +bash scripts/infer_efficientnet_v2_s_fp16_accuracy.sh +# Performance +bash scripts/infer_efficientnet_v2_s_fp16_performance.sh +``` + +## Results + +| Model | BatchSize | Precision | FPS | Top-1(%) | Top-5(%) | +| ----------------- | --------- | --------- | -------- | -------- | -------- | +| Efficientnet_v2_s | 32 | FP16 | 2357.457 | 81.290 | 95.242 | diff --git a/models/cv/classification/efficientnet_v2_s/igie/build_engine.py b/models/cv/classification/efficientnet_v2_s/igie/build_engine.py new file mode 100644 index 00000000..d3626ae7 --- /dev/null +++ b/models/cv/classification/efficientnet_v2_s/igie/build_engine.py @@ -0,0 +1,73 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import tvm +import argparse +from tvm import relay +from tvm.relay.import_model import import_model_to_igie + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--model_path", + type=str, + required=True, + help="original model path.") + + parser.add_argument("--engine_path", + type=str, + required=True, + help="igie export engine path.") + + parser.add_argument("--input", + type=str, + required=True, + help=""" + input info of the model, format should be: + input_name:input_shape + eg: --input input:1,3,224,224. + """) + + parser.add_argument("--precision", + type=str, + choices=["fp32", "fp16", "int8"], + required=True, + help="model inference precision.") + + args = parser.parse_args() + + return args + +def main(): + args = parse_args() + + # get input valueinfo + input_name, input_shape = args.input.split(":") + shape = tuple([int(s) for s in input_shape.split(",")]) + input_dict = {input_name: shape} + + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + + mod, params = import_model_to_igie(args.model_path, input_dict, backend="igie") + + # build engine + lib = tvm.relay.build(mod, target=target, params=params, precision=args.precision) + + # export engine + lib.export_library(args.engine_path) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/efficientnet_v2_s/igie/export.py b/models/cv/classification/efficientnet_v2_s/igie/export.py new file mode 100644 index 00000000..63b1b4c8 --- /dev/null +++ b/models/cv/classification/efficientnet_v2_s/igie/export.py @@ -0,0 +1,61 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import torch +import torchvision +import argparse + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--weight", + type=str, + required=True, + help="pytorch model weight.") + + parser.add_argument("--output", + type=str, + required=True, + help="export onnx model path.") + + args = parser.parse_args() + return args + +def main(): + args = parse_args() + + model = torchvision.models.efficientnet_v2_s() + model.load_state_dict(torch.load(args.weight)) + model.eval() + + input_names = ['input'] + output_names = ['output'] + dynamic_axes = {'input': {0: '-1'}, 'output': {0: '-1'}} + dummy_input = torch.randn(1, 3, 224, 224) + + torch.onnx.export( + model, + dummy_input, + args.output, + input_names = input_names, + dynamic_axes = dynamic_axes, + output_names = output_names, + opset_version=13 + ) + + print("Export onnx model successfully! ") + +if __name__ == "__main__": + main() diff --git a/models/cv/classification/efficientnet_v2_s/igie/inference.py b/models/cv/classification/efficientnet_v2_s/igie/inference.py new file mode 100644 index 00000000..3aef3ec7 --- /dev/null +++ b/models/cv/classification/efficientnet_v2_s/igie/inference.py @@ -0,0 +1,186 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +import argparse +import tvm +import torch +import torchvision +import numpy as np +from tvm import relay +from tqdm import tqdm +from torchvision import transforms +from torchvision.transforms.functional import InterpolationMode + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--engine", + type=str, + required=True, + help="igie engine path.") + + parser.add_argument("--batchsize", + type=int, + required=True, + help="inference batch size.") + + parser.add_argument("--datasets", + type=str, + required=True, + help="datasets path.") + + parser.add_argument("--input_name", + type=str, + required=True, + help="input name of the model.") + + parser.add_argument("--warmup", + type=int, + default=3, + help="number of warmup before test.") + + parser.add_argument("--num_workers", + type=int, + default=16, + help="number of workers used in pytorch dataloader.") + + parser.add_argument("--acc_target", + type=float, + default=None, + help="Model inference Accuracy target.") + + parser.add_argument("--fps_target", + type=float, + default=None, + help="Model inference FPS target.") + + parser.add_argument("--perf_only", + type=bool, + default=False, + help="Run performance test only") + + args = parser.parse_args() + + return args + +def get_dataloader(data_path, batch_size, num_workers): + dataset = torchvision.datasets.ImageFolder( + data_path, + transforms.Compose( + [ + transforms.Resize(256, interpolation=InterpolationMode.BILINEAR), + transforms.CenterCrop(224), + transforms.PILToTensor(), + transforms.ConvertImageDtype(torch.float), + transforms.Normalize( + mean=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225) + ) + ] + ) + ) + + dataloader = torch.utils.data.DataLoader(dataset, batch_size, num_workers=num_workers) + + return dataloader + +def get_topk_accuracy(pred, label): + if isinstance(pred, np.ndarray): + pred = torch.from_numpy(pred) + + if isinstance(label, np.ndarray): + label = torch.from_numpy(label) + + top1_acc = 0 + top5_acc = 0 + for idx in range(len(label)): + label_value = label[idx] + if label_value == torch.topk(pred[idx].float(), 1).indices.data: + top1_acc += 1 + top5_acc += 1 + + elif label_value in torch.topk(pred[idx].float(), 5).indices.data: + top5_acc += 1 + + return top1_acc, top5_acc + +def main(): + args = parse_args() + + batch_size = args.batchsize + + # create iluvatar target & device + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + device = tvm.device(target.kind.name, 0) + + # load engine + lib = tvm.runtime.load_module(args.engine) + + # create runtime from engine + module = tvm.contrib.graph_executor.GraphModule(lib["default"](device)) + + # just run perf test + if args.perf_only: + ftimer = module.module.time_evaluator("run", device, number=100, repeat=1) + prof_res = np.array(ftimer().results) * 1000 + fps = batch_size * 1000 / np.mean(prof_res) + print(f"\n* Mean inference time: {np.mean(prof_res):.3f} ms, Mean fps: {fps:.3f}") + else: + # warm up + for _ in range(args.warmup): + module.run() + + # get dataloader + dataloader = get_dataloader(args.datasets, batch_size, args.num_workers) + + top1_acc = 0 + top5_acc = 0 + total_num = 0 + + for image, label in tqdm(dataloader): + + # pad the last batch + pad_batch = len(image) != batch_size + + if pad_batch: + origin_size = len(image) + image = np.resize(image, (batch_size, *image.shape[1:])) + + module.set_input(args.input_name, tvm.nd.array(image, device)) + + # run inference + module.run() + + pred = module.get_output(0).asnumpy() + + if pad_batch: + pred = pred[:origin_size] + + # get batch accuracy + batch_top1_acc, batch_top5_acc = get_topk_accuracy(pred, label) + + top1_acc += batch_top1_acc + top5_acc += batch_top5_acc + total_num += batch_size + + result_stat = {} + result_stat["acc@1"] = round(top1_acc / total_num * 100.0, 3) + result_stat["acc@5"] = round(top5_acc / total_num * 100.0, 3) + + print(f"\n* Top1 acc: {result_stat['acc@1']} %, Top5 acc: {result_stat['acc@5']} %") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/efficientnet_v2_s/igie/scripts/infer_efficientnet_v2_s_fp16_accuracy.sh b/models/cv/classification/efficientnet_v2_s/igie/scripts/infer_efficientnet_v2_s_fp16_accuracy.sh new file mode 100644 index 00000000..2e7d1133 --- /dev/null +++ b/models/cv/classification/efficientnet_v2_s/igie/scripts/infer_efficientnet_v2_s_fp16_accuracy.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="efficientnet_v2_s.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path efficientnet_v2_s_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine efficientnet_v2_s_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ No newline at end of file diff --git a/models/cv/classification/efficientnet_v2_s/igie/scripts/infer_efficientnet_v2_s_fp16_performance.sh b/models/cv/classification/efficientnet_v2_s/igie/scripts/infer_efficientnet_v2_s_fp16_performance.sh new file mode 100644 index 00000000..4c67dd99 --- /dev/null +++ b/models/cv/classification/efficientnet_v2_s/igie/scripts/infer_efficientnet_v2_s_fp16_performance.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="efficientnet_v2_s.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path efficientnet_v2_s_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine efficientnet_v2_s_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ + --perf_only True \ No newline at end of file -- Gitee From c587a7b5ece466429e432a34de96265d32b80c44 Mon Sep 17 00:00:00 2001 From: YoungPeng Date: Wed, 9 Oct 2024 11:31:30 +0800 Subject: [PATCH 05/10] Add: mnasnet0_5 inference script. --- .../classification/mnasnet0_5/igie/README.md | 48 +++++ .../mnasnet0_5/igie/build_engine.py | 73 +++++++ .../classification/mnasnet0_5/igie/export.py | 61 ++++++ .../mnasnet0_5/igie/inference.py | 186 ++++++++++++++++++ .../scripts/infer_mnasnet0_5_fp16_accuracy.sh | 35 ++++ .../infer_mnasnet0_5_fp16_performance.sh | 36 ++++ 6 files changed, 439 insertions(+) create mode 100644 models/cv/classification/mnasnet0_5/igie/README.md create mode 100644 models/cv/classification/mnasnet0_5/igie/build_engine.py create mode 100644 models/cv/classification/mnasnet0_5/igie/export.py create mode 100644 models/cv/classification/mnasnet0_5/igie/inference.py create mode 100644 models/cv/classification/mnasnet0_5/igie/scripts/infer_mnasnet0_5_fp16_accuracy.sh create mode 100644 models/cv/classification/mnasnet0_5/igie/scripts/infer_mnasnet0_5_fp16_performance.sh diff --git a/models/cv/classification/mnasnet0_5/igie/README.md b/models/cv/classification/mnasnet0_5/igie/README.md new file mode 100644 index 00000000..3055a187 --- /dev/null +++ b/models/cv/classification/mnasnet0_5/igie/README.md @@ -0,0 +1,48 @@ +# MNASNet0_5 + +## Description + +MNASNet0_5 is a neural network architecture optimized for mobile devices, designed through neural architecture search technology. It is characterized by high efficiency and excellent accuracy, offering 50% higher accuracy than MobileNetV2 while maintaining low latency and memory usage. MNASNet0_5 widely uses depthwise separable convolutions, supports multi-scale inputs, and demonstrates good robustness, making it suitable for real-time image recognition tasks in resource-constrained environments. + + +## Setup + +### Install + +```bash +pip3 install onnx +pip3 install tqdm +``` + +### Download + +Pretrained model: + +Dataset: to download the validation dataset. + +### Model Conversion + +```bash +python3 export.py --weight mnasnet0.5_top1_67.823-3ffadce67e.pth --output mnasnet0_5.onnx +``` + +## Inference + +```bash +export DATASETS_DIR=/Path/to/imagenet_val/ +``` + +### FP16 + +```bash +# Accuracy +bash scripts/infer_mnasnet0_5_fp16_accuracy.sh +# Performance +bash scripts/infer_mnasnet0_5_fp16_performance.sh +``` + +## Results + +| Model | BatchSize | Precision | FPS | Top-1(%) | Top-5(%) | +| ----------------- | --------- | --------- | -------- | -------- | -------- | +| MnasNet0_5 | 32 | FP16 | 7933.980 | 67.748 | 87.452 | diff --git a/models/cv/classification/mnasnet0_5/igie/build_engine.py b/models/cv/classification/mnasnet0_5/igie/build_engine.py new file mode 100644 index 00000000..d3626ae7 --- /dev/null +++ b/models/cv/classification/mnasnet0_5/igie/build_engine.py @@ -0,0 +1,73 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import tvm +import argparse +from tvm import relay +from tvm.relay.import_model import import_model_to_igie + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--model_path", + type=str, + required=True, + help="original model path.") + + parser.add_argument("--engine_path", + type=str, + required=True, + help="igie export engine path.") + + parser.add_argument("--input", + type=str, + required=True, + help=""" + input info of the model, format should be: + input_name:input_shape + eg: --input input:1,3,224,224. + """) + + parser.add_argument("--precision", + type=str, + choices=["fp32", "fp16", "int8"], + required=True, + help="model inference precision.") + + args = parser.parse_args() + + return args + +def main(): + args = parse_args() + + # get input valueinfo + input_name, input_shape = args.input.split(":") + shape = tuple([int(s) for s in input_shape.split(",")]) + input_dict = {input_name: shape} + + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + + mod, params = import_model_to_igie(args.model_path, input_dict, backend="igie") + + # build engine + lib = tvm.relay.build(mod, target=target, params=params, precision=args.precision) + + # export engine + lib.export_library(args.engine_path) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/mnasnet0_5/igie/export.py b/models/cv/classification/mnasnet0_5/igie/export.py new file mode 100644 index 00000000..bd48e206 --- /dev/null +++ b/models/cv/classification/mnasnet0_5/igie/export.py @@ -0,0 +1,61 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import torch +import torchvision +import argparse + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--weight", + type=str, + required=True, + help="pytorch model weight.") + + parser.add_argument("--output", + type=str, + required=True, + help="export onnx model path.") + + args = parser.parse_args() + return args + +def main(): + args = parse_args() + + model = torchvision.models.mnasnet0_5() + model.load_state_dict(torch.load(args.weight)) + model.eval() + + input_names = ['input'] + output_names = ['output'] + dynamic_axes = {'input': {0: '-1'}, 'output': {0: '-1'}} + dummy_input = torch.randn(1, 3, 224, 224) + + torch.onnx.export( + model, + dummy_input, + args.output, + input_names = input_names, + dynamic_axes = dynamic_axes, + output_names = output_names, + opset_version=13 + ) + + print("Export onnx model successfully! ") + +if __name__ == "__main__": + main() diff --git a/models/cv/classification/mnasnet0_5/igie/inference.py b/models/cv/classification/mnasnet0_5/igie/inference.py new file mode 100644 index 00000000..3aef3ec7 --- /dev/null +++ b/models/cv/classification/mnasnet0_5/igie/inference.py @@ -0,0 +1,186 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +import argparse +import tvm +import torch +import torchvision +import numpy as np +from tvm import relay +from tqdm import tqdm +from torchvision import transforms +from torchvision.transforms.functional import InterpolationMode + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--engine", + type=str, + required=True, + help="igie engine path.") + + parser.add_argument("--batchsize", + type=int, + required=True, + help="inference batch size.") + + parser.add_argument("--datasets", + type=str, + required=True, + help="datasets path.") + + parser.add_argument("--input_name", + type=str, + required=True, + help="input name of the model.") + + parser.add_argument("--warmup", + type=int, + default=3, + help="number of warmup before test.") + + parser.add_argument("--num_workers", + type=int, + default=16, + help="number of workers used in pytorch dataloader.") + + parser.add_argument("--acc_target", + type=float, + default=None, + help="Model inference Accuracy target.") + + parser.add_argument("--fps_target", + type=float, + default=None, + help="Model inference FPS target.") + + parser.add_argument("--perf_only", + type=bool, + default=False, + help="Run performance test only") + + args = parser.parse_args() + + return args + +def get_dataloader(data_path, batch_size, num_workers): + dataset = torchvision.datasets.ImageFolder( + data_path, + transforms.Compose( + [ + transforms.Resize(256, interpolation=InterpolationMode.BILINEAR), + transforms.CenterCrop(224), + transforms.PILToTensor(), + transforms.ConvertImageDtype(torch.float), + transforms.Normalize( + mean=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225) + ) + ] + ) + ) + + dataloader = torch.utils.data.DataLoader(dataset, batch_size, num_workers=num_workers) + + return dataloader + +def get_topk_accuracy(pred, label): + if isinstance(pred, np.ndarray): + pred = torch.from_numpy(pred) + + if isinstance(label, np.ndarray): + label = torch.from_numpy(label) + + top1_acc = 0 + top5_acc = 0 + for idx in range(len(label)): + label_value = label[idx] + if label_value == torch.topk(pred[idx].float(), 1).indices.data: + top1_acc += 1 + top5_acc += 1 + + elif label_value in torch.topk(pred[idx].float(), 5).indices.data: + top5_acc += 1 + + return top1_acc, top5_acc + +def main(): + args = parse_args() + + batch_size = args.batchsize + + # create iluvatar target & device + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + device = tvm.device(target.kind.name, 0) + + # load engine + lib = tvm.runtime.load_module(args.engine) + + # create runtime from engine + module = tvm.contrib.graph_executor.GraphModule(lib["default"](device)) + + # just run perf test + if args.perf_only: + ftimer = module.module.time_evaluator("run", device, number=100, repeat=1) + prof_res = np.array(ftimer().results) * 1000 + fps = batch_size * 1000 / np.mean(prof_res) + print(f"\n* Mean inference time: {np.mean(prof_res):.3f} ms, Mean fps: {fps:.3f}") + else: + # warm up + for _ in range(args.warmup): + module.run() + + # get dataloader + dataloader = get_dataloader(args.datasets, batch_size, args.num_workers) + + top1_acc = 0 + top5_acc = 0 + total_num = 0 + + for image, label in tqdm(dataloader): + + # pad the last batch + pad_batch = len(image) != batch_size + + if pad_batch: + origin_size = len(image) + image = np.resize(image, (batch_size, *image.shape[1:])) + + module.set_input(args.input_name, tvm.nd.array(image, device)) + + # run inference + module.run() + + pred = module.get_output(0).asnumpy() + + if pad_batch: + pred = pred[:origin_size] + + # get batch accuracy + batch_top1_acc, batch_top5_acc = get_topk_accuracy(pred, label) + + top1_acc += batch_top1_acc + top5_acc += batch_top5_acc + total_num += batch_size + + result_stat = {} + result_stat["acc@1"] = round(top1_acc / total_num * 100.0, 3) + result_stat["acc@5"] = round(top5_acc / total_num * 100.0, 3) + + print(f"\n* Top1 acc: {result_stat['acc@1']} %, Top5 acc: {result_stat['acc@5']} %") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/mnasnet0_5/igie/scripts/infer_mnasnet0_5_fp16_accuracy.sh b/models/cv/classification/mnasnet0_5/igie/scripts/infer_mnasnet0_5_fp16_accuracy.sh new file mode 100644 index 00000000..a1c3b37a --- /dev/null +++ b/models/cv/classification/mnasnet0_5/igie/scripts/infer_mnasnet0_5_fp16_accuracy.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="mnasnet0_5.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path mnasnet0_5_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine mnasnet0_5_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ No newline at end of file diff --git a/models/cv/classification/mnasnet0_5/igie/scripts/infer_mnasnet0_5_fp16_performance.sh b/models/cv/classification/mnasnet0_5/igie/scripts/infer_mnasnet0_5_fp16_performance.sh new file mode 100644 index 00000000..89271d33 --- /dev/null +++ b/models/cv/classification/mnasnet0_5/igie/scripts/infer_mnasnet0_5_fp16_performance.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="mnasnet0_5.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path mnasnet0_5_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine mnasnet0_5_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ + --perf_only True \ No newline at end of file -- Gitee From 65c0c21f13e720d35a40d74f0211816624d1c160 Mon Sep 17 00:00:00 2001 From: YoungPeng Date: Wed, 9 Oct 2024 14:15:43 +0800 Subject: [PATCH 06/10] Add: regnet_y_1_6gf inference script. --- .../regnet_y_1_6gf/igie/README.md | 47 +++++ .../regnet_y_1_6gf/igie/build_engine.py | 73 +++++++ .../regnet_y_1_6gf/igie/export.py | 61 ++++++ .../regnet_y_1_6gf/igie/inference.py | 186 ++++++++++++++++++ .../infer_regnet_y_1_6gf_fp16_accuracy.sh | 35 ++++ .../infer_regnet_y_1_6gf_fp16_performance.sh | 36 ++++ 6 files changed, 438 insertions(+) create mode 100644 models/cv/classification/regnet_y_1_6gf/igie/README.md create mode 100644 models/cv/classification/regnet_y_1_6gf/igie/build_engine.py create mode 100644 models/cv/classification/regnet_y_1_6gf/igie/export.py create mode 100644 models/cv/classification/regnet_y_1_6gf/igie/inference.py create mode 100644 models/cv/classification/regnet_y_1_6gf/igie/scripts/infer_regnet_y_1_6gf_fp16_accuracy.sh create mode 100644 models/cv/classification/regnet_y_1_6gf/igie/scripts/infer_regnet_y_1_6gf_fp16_performance.sh diff --git a/models/cv/classification/regnet_y_1_6gf/igie/README.md b/models/cv/classification/regnet_y_1_6gf/igie/README.md new file mode 100644 index 00000000..3f96b09b --- /dev/null +++ b/models/cv/classification/regnet_y_1_6gf/igie/README.md @@ -0,0 +1,47 @@ +# RegNet_y_1_6gf + +## Description + +RegNet is a family of models designed for image classification tasks, as described in the paper "Designing Network Design Spaces". The RegNet design space provides simple and fast networks that work well across a wide range of computational budgets.The architecture of RegNet models is based on the principle of designing network design spaces, which allows for a more systematic exploration of possible network architectures. This makes it easier to understand and modify the architecture.RegNet_y_1_6gf is a specific model within the RegNet family, designed for image classification tasks. + +## Setup + +### Install + +```bash +pip3 install onnx +pip3 install tqdm +``` + +### Download + +Pretrained model: + +Dataset: to download the validation dataset. + +### Model Conversion + +```bash +python3 export.py --weight regnet_y_1_6gf-b11a554e.pth --output regnet_y_1_6gf.onnx +``` + +## Inference + +```bash +export DATASETS_DIR=/Path/to/imagenet_val/ +``` + +### FP16 + +```bash +# Accuracy +bash scripts/infer_regnet_y_1_6gf_fp16_accuracy.sh +# Performance +bash scripts/infer_regnet_y_1_6gf_fp16_performance.sh +``` + +## Results + +Model |BatchSize |Precision |FPS |Top-1(%) |Top-5(%) +------------------|-----------|----------|---------|---------|-------- +RegNet_y_1_6gf | 32 | FP16 | 1785.44 | 77.933 | 93.948 diff --git a/models/cv/classification/regnet_y_1_6gf/igie/build_engine.py b/models/cv/classification/regnet_y_1_6gf/igie/build_engine.py new file mode 100644 index 00000000..d3626ae7 --- /dev/null +++ b/models/cv/classification/regnet_y_1_6gf/igie/build_engine.py @@ -0,0 +1,73 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import tvm +import argparse +from tvm import relay +from tvm.relay.import_model import import_model_to_igie + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--model_path", + type=str, + required=True, + help="original model path.") + + parser.add_argument("--engine_path", + type=str, + required=True, + help="igie export engine path.") + + parser.add_argument("--input", + type=str, + required=True, + help=""" + input info of the model, format should be: + input_name:input_shape + eg: --input input:1,3,224,224. + """) + + parser.add_argument("--precision", + type=str, + choices=["fp32", "fp16", "int8"], + required=True, + help="model inference precision.") + + args = parser.parse_args() + + return args + +def main(): + args = parse_args() + + # get input valueinfo + input_name, input_shape = args.input.split(":") + shape = tuple([int(s) for s in input_shape.split(",")]) + input_dict = {input_name: shape} + + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + + mod, params = import_model_to_igie(args.model_path, input_dict, backend="igie") + + # build engine + lib = tvm.relay.build(mod, target=target, params=params, precision=args.precision) + + # export engine + lib.export_library(args.engine_path) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/regnet_y_1_6gf/igie/export.py b/models/cv/classification/regnet_y_1_6gf/igie/export.py new file mode 100644 index 00000000..f2a9f0a2 --- /dev/null +++ b/models/cv/classification/regnet_y_1_6gf/igie/export.py @@ -0,0 +1,61 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import torch +import torchvision +import argparse + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--weight", + type=str, + required=True, + help="pytorch model weight.") + + parser.add_argument("--output", + type=str, + required=True, + help="export onnx model path.") + + args = parser.parse_args() + return args + +def main(): + args = parse_args() + + model = torchvision.models.regnet_y_1_6gf() + model.load_state_dict(torch.load(args.weight)) + model.eval() + + input_names = ['input'] + output_names = ['output'] + dynamic_axes = {'input': {0: '-1'}, 'output': {0: '-1'}} + dummy_input = torch.randn(1, 3, 224, 224) + + torch.onnx.export( + model, + dummy_input, + args.output, + input_names = input_names, + dynamic_axes = dynamic_axes, + output_names = output_names, + opset_version=13 + ) + + print("Export onnx model successfully! ") + +if __name__ == "__main__": + main() diff --git a/models/cv/classification/regnet_y_1_6gf/igie/inference.py b/models/cv/classification/regnet_y_1_6gf/igie/inference.py new file mode 100644 index 00000000..3aef3ec7 --- /dev/null +++ b/models/cv/classification/regnet_y_1_6gf/igie/inference.py @@ -0,0 +1,186 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +import argparse +import tvm +import torch +import torchvision +import numpy as np +from tvm import relay +from tqdm import tqdm +from torchvision import transforms +from torchvision.transforms.functional import InterpolationMode + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--engine", + type=str, + required=True, + help="igie engine path.") + + parser.add_argument("--batchsize", + type=int, + required=True, + help="inference batch size.") + + parser.add_argument("--datasets", + type=str, + required=True, + help="datasets path.") + + parser.add_argument("--input_name", + type=str, + required=True, + help="input name of the model.") + + parser.add_argument("--warmup", + type=int, + default=3, + help="number of warmup before test.") + + parser.add_argument("--num_workers", + type=int, + default=16, + help="number of workers used in pytorch dataloader.") + + parser.add_argument("--acc_target", + type=float, + default=None, + help="Model inference Accuracy target.") + + parser.add_argument("--fps_target", + type=float, + default=None, + help="Model inference FPS target.") + + parser.add_argument("--perf_only", + type=bool, + default=False, + help="Run performance test only") + + args = parser.parse_args() + + return args + +def get_dataloader(data_path, batch_size, num_workers): + dataset = torchvision.datasets.ImageFolder( + data_path, + transforms.Compose( + [ + transforms.Resize(256, interpolation=InterpolationMode.BILINEAR), + transforms.CenterCrop(224), + transforms.PILToTensor(), + transforms.ConvertImageDtype(torch.float), + transforms.Normalize( + mean=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225) + ) + ] + ) + ) + + dataloader = torch.utils.data.DataLoader(dataset, batch_size, num_workers=num_workers) + + return dataloader + +def get_topk_accuracy(pred, label): + if isinstance(pred, np.ndarray): + pred = torch.from_numpy(pred) + + if isinstance(label, np.ndarray): + label = torch.from_numpy(label) + + top1_acc = 0 + top5_acc = 0 + for idx in range(len(label)): + label_value = label[idx] + if label_value == torch.topk(pred[idx].float(), 1).indices.data: + top1_acc += 1 + top5_acc += 1 + + elif label_value in torch.topk(pred[idx].float(), 5).indices.data: + top5_acc += 1 + + return top1_acc, top5_acc + +def main(): + args = parse_args() + + batch_size = args.batchsize + + # create iluvatar target & device + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + device = tvm.device(target.kind.name, 0) + + # load engine + lib = tvm.runtime.load_module(args.engine) + + # create runtime from engine + module = tvm.contrib.graph_executor.GraphModule(lib["default"](device)) + + # just run perf test + if args.perf_only: + ftimer = module.module.time_evaluator("run", device, number=100, repeat=1) + prof_res = np.array(ftimer().results) * 1000 + fps = batch_size * 1000 / np.mean(prof_res) + print(f"\n* Mean inference time: {np.mean(prof_res):.3f} ms, Mean fps: {fps:.3f}") + else: + # warm up + for _ in range(args.warmup): + module.run() + + # get dataloader + dataloader = get_dataloader(args.datasets, batch_size, args.num_workers) + + top1_acc = 0 + top5_acc = 0 + total_num = 0 + + for image, label in tqdm(dataloader): + + # pad the last batch + pad_batch = len(image) != batch_size + + if pad_batch: + origin_size = len(image) + image = np.resize(image, (batch_size, *image.shape[1:])) + + module.set_input(args.input_name, tvm.nd.array(image, device)) + + # run inference + module.run() + + pred = module.get_output(0).asnumpy() + + if pad_batch: + pred = pred[:origin_size] + + # get batch accuracy + batch_top1_acc, batch_top5_acc = get_topk_accuracy(pred, label) + + top1_acc += batch_top1_acc + top5_acc += batch_top5_acc + total_num += batch_size + + result_stat = {} + result_stat["acc@1"] = round(top1_acc / total_num * 100.0, 3) + result_stat["acc@5"] = round(top5_acc / total_num * 100.0, 3) + + print(f"\n* Top1 acc: {result_stat['acc@1']} %, Top5 acc: {result_stat['acc@5']} %") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/regnet_y_1_6gf/igie/scripts/infer_regnet_y_1_6gf_fp16_accuracy.sh b/models/cv/classification/regnet_y_1_6gf/igie/scripts/infer_regnet_y_1_6gf_fp16_accuracy.sh new file mode 100644 index 00000000..62d9cc11 --- /dev/null +++ b/models/cv/classification/regnet_y_1_6gf/igie/scripts/infer_regnet_y_1_6gf_fp16_accuracy.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="regnet_y_1_6gf.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path regnet_y_1_6gf_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine regnet_y_1_6gf_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ No newline at end of file diff --git a/models/cv/classification/regnet_y_1_6gf/igie/scripts/infer_regnet_y_1_6gf_fp16_performance.sh b/models/cv/classification/regnet_y_1_6gf/igie/scripts/infer_regnet_y_1_6gf_fp16_performance.sh new file mode 100644 index 00000000..afe3a450 --- /dev/null +++ b/models/cv/classification/regnet_y_1_6gf/igie/scripts/infer_regnet_y_1_6gf_fp16_performance.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="regnet_y_1_6gf.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path regnet_y_1_6gf_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine regnet_y_1_6gf_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ + --perf_only True \ No newline at end of file -- Gitee From 8cbaab5f888edc43085321e2e861a17aee187710 Mon Sep 17 00:00:00 2001 From: YoungPeng Date: Wed, 9 Oct 2024 15:04:53 +0800 Subject: [PATCH 07/10] Add: resnext101_64x4d inference script. --- .../resnext101_64x4d/igie/README.md | 47 +++++ .../resnext101_64x4d/igie/build_engine.py | 73 +++++++ .../resnext101_64x4d/igie/export.py | 61 ++++++ .../resnext101_64x4d/igie/inference.py | 186 ++++++++++++++++++ .../infer_resnext101_64x4d_fp16_accuracy.sh | 35 ++++ ...infer_resnext101_64x4d_fp16_performance.sh | 36 ++++ 6 files changed, 438 insertions(+) create mode 100644 models/cv/classification/resnext101_64x4d/igie/README.md create mode 100644 models/cv/classification/resnext101_64x4d/igie/build_engine.py create mode 100644 models/cv/classification/resnext101_64x4d/igie/export.py create mode 100644 models/cv/classification/resnext101_64x4d/igie/inference.py create mode 100644 models/cv/classification/resnext101_64x4d/igie/scripts/infer_resnext101_64x4d_fp16_accuracy.sh create mode 100644 models/cv/classification/resnext101_64x4d/igie/scripts/infer_resnext101_64x4d_fp16_performance.sh diff --git a/models/cv/classification/resnext101_64x4d/igie/README.md b/models/cv/classification/resnext101_64x4d/igie/README.md new file mode 100644 index 00000000..5b9846f2 --- /dev/null +++ b/models/cv/classification/resnext101_64x4d/igie/README.md @@ -0,0 +1,47 @@ +# ResNext101_64x4d + +## Description + +The ResNeXt101_64x4d is a deep learning model based on the deep residual network architecture, which enhances performance and efficiency through the use of grouped convolutions. With a depth of 101 layers and 64 filter groups, it is particularly suited for complex image recognition tasks. While maintaining excellent accuracy, it can adapt to various input sizes + +## Setup + +### Install + +```bash +pip3 install onnx +pip3 install tqdm +``` + +### Download + +Pretrained model: + +Dataset: to download the validation dataset. + +### Model Conversion + +```bash +python3 export.py --weight resnext101_64x4d-173b62eb.pth --output resnext101_64x4d.onnx +``` + +## Inference + +```bash +export DATASETS_DIR=/Path/to/imagenet_val/ +``` + +### FP16 + +```bash +# Accuracy +bash scripts/infer_resnext101_64x4d_fp16_accuracy.sh +# Performance +bash scripts/infer_resnext101_64x4d_fp16_performance.sh +``` + +## Results + +Model |BatchSize |Precision |FPS |Top-1(%) |Top-5(%) +----------------|-----------|----------|---------|----------|-------- +ResNext101_64x4d| 32 | FP16 | 663.13 | 82.953 | 96.221 diff --git a/models/cv/classification/resnext101_64x4d/igie/build_engine.py b/models/cv/classification/resnext101_64x4d/igie/build_engine.py new file mode 100644 index 00000000..d3626ae7 --- /dev/null +++ b/models/cv/classification/resnext101_64x4d/igie/build_engine.py @@ -0,0 +1,73 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import tvm +import argparse +from tvm import relay +from tvm.relay.import_model import import_model_to_igie + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--model_path", + type=str, + required=True, + help="original model path.") + + parser.add_argument("--engine_path", + type=str, + required=True, + help="igie export engine path.") + + parser.add_argument("--input", + type=str, + required=True, + help=""" + input info of the model, format should be: + input_name:input_shape + eg: --input input:1,3,224,224. + """) + + parser.add_argument("--precision", + type=str, + choices=["fp32", "fp16", "int8"], + required=True, + help="model inference precision.") + + args = parser.parse_args() + + return args + +def main(): + args = parse_args() + + # get input valueinfo + input_name, input_shape = args.input.split(":") + shape = tuple([int(s) for s in input_shape.split(",")]) + input_dict = {input_name: shape} + + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + + mod, params = import_model_to_igie(args.model_path, input_dict, backend="igie") + + # build engine + lib = tvm.relay.build(mod, target=target, params=params, precision=args.precision) + + # export engine + lib.export_library(args.engine_path) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/resnext101_64x4d/igie/export.py b/models/cv/classification/resnext101_64x4d/igie/export.py new file mode 100644 index 00000000..43a20fca --- /dev/null +++ b/models/cv/classification/resnext101_64x4d/igie/export.py @@ -0,0 +1,61 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import torch +import torchvision +import argparse + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--weight", + type=str, + required=True, + help="pytorch model weight.") + + parser.add_argument("--output", + type=str, + required=True, + help="export onnx model path.") + + args = parser.parse_args() + return args + +def main(): + args = parse_args() + + model = torchvision.models.resnext101_64x4d() + model.load_state_dict(torch.load(args.weight)) + model.eval() + + input_names = ['input'] + output_names = ['output'] + dynamic_axes = {'input': {0: '-1'}, 'output': {0: '-1'}} + dummy_input = torch.randn(1, 3, 224, 224) + + torch.onnx.export( + model, + dummy_input, + args.output, + input_names = input_names, + dynamic_axes = dynamic_axes, + output_names = output_names, + opset_version=13 + ) + + print("Export onnx model successfully! ") + +if __name__ == "__main__": + main() diff --git a/models/cv/classification/resnext101_64x4d/igie/inference.py b/models/cv/classification/resnext101_64x4d/igie/inference.py new file mode 100644 index 00000000..3aef3ec7 --- /dev/null +++ b/models/cv/classification/resnext101_64x4d/igie/inference.py @@ -0,0 +1,186 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +import argparse +import tvm +import torch +import torchvision +import numpy as np +from tvm import relay +from tqdm import tqdm +from torchvision import transforms +from torchvision.transforms.functional import InterpolationMode + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--engine", + type=str, + required=True, + help="igie engine path.") + + parser.add_argument("--batchsize", + type=int, + required=True, + help="inference batch size.") + + parser.add_argument("--datasets", + type=str, + required=True, + help="datasets path.") + + parser.add_argument("--input_name", + type=str, + required=True, + help="input name of the model.") + + parser.add_argument("--warmup", + type=int, + default=3, + help="number of warmup before test.") + + parser.add_argument("--num_workers", + type=int, + default=16, + help="number of workers used in pytorch dataloader.") + + parser.add_argument("--acc_target", + type=float, + default=None, + help="Model inference Accuracy target.") + + parser.add_argument("--fps_target", + type=float, + default=None, + help="Model inference FPS target.") + + parser.add_argument("--perf_only", + type=bool, + default=False, + help="Run performance test only") + + args = parser.parse_args() + + return args + +def get_dataloader(data_path, batch_size, num_workers): + dataset = torchvision.datasets.ImageFolder( + data_path, + transforms.Compose( + [ + transforms.Resize(256, interpolation=InterpolationMode.BILINEAR), + transforms.CenterCrop(224), + transforms.PILToTensor(), + transforms.ConvertImageDtype(torch.float), + transforms.Normalize( + mean=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225) + ) + ] + ) + ) + + dataloader = torch.utils.data.DataLoader(dataset, batch_size, num_workers=num_workers) + + return dataloader + +def get_topk_accuracy(pred, label): + if isinstance(pred, np.ndarray): + pred = torch.from_numpy(pred) + + if isinstance(label, np.ndarray): + label = torch.from_numpy(label) + + top1_acc = 0 + top5_acc = 0 + for idx in range(len(label)): + label_value = label[idx] + if label_value == torch.topk(pred[idx].float(), 1).indices.data: + top1_acc += 1 + top5_acc += 1 + + elif label_value in torch.topk(pred[idx].float(), 5).indices.data: + top5_acc += 1 + + return top1_acc, top5_acc + +def main(): + args = parse_args() + + batch_size = args.batchsize + + # create iluvatar target & device + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + device = tvm.device(target.kind.name, 0) + + # load engine + lib = tvm.runtime.load_module(args.engine) + + # create runtime from engine + module = tvm.contrib.graph_executor.GraphModule(lib["default"](device)) + + # just run perf test + if args.perf_only: + ftimer = module.module.time_evaluator("run", device, number=100, repeat=1) + prof_res = np.array(ftimer().results) * 1000 + fps = batch_size * 1000 / np.mean(prof_res) + print(f"\n* Mean inference time: {np.mean(prof_res):.3f} ms, Mean fps: {fps:.3f}") + else: + # warm up + for _ in range(args.warmup): + module.run() + + # get dataloader + dataloader = get_dataloader(args.datasets, batch_size, args.num_workers) + + top1_acc = 0 + top5_acc = 0 + total_num = 0 + + for image, label in tqdm(dataloader): + + # pad the last batch + pad_batch = len(image) != batch_size + + if pad_batch: + origin_size = len(image) + image = np.resize(image, (batch_size, *image.shape[1:])) + + module.set_input(args.input_name, tvm.nd.array(image, device)) + + # run inference + module.run() + + pred = module.get_output(0).asnumpy() + + if pad_batch: + pred = pred[:origin_size] + + # get batch accuracy + batch_top1_acc, batch_top5_acc = get_topk_accuracy(pred, label) + + top1_acc += batch_top1_acc + top5_acc += batch_top5_acc + total_num += batch_size + + result_stat = {} + result_stat["acc@1"] = round(top1_acc / total_num * 100.0, 3) + result_stat["acc@5"] = round(top5_acc / total_num * 100.0, 3) + + print(f"\n* Top1 acc: {result_stat['acc@1']} %, Top5 acc: {result_stat['acc@5']} %") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/resnext101_64x4d/igie/scripts/infer_resnext101_64x4d_fp16_accuracy.sh b/models/cv/classification/resnext101_64x4d/igie/scripts/infer_resnext101_64x4d_fp16_accuracy.sh new file mode 100644 index 00000000..edcb1a00 --- /dev/null +++ b/models/cv/classification/resnext101_64x4d/igie/scripts/infer_resnext101_64x4d_fp16_accuracy.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="resnext101_64x4d.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path resnext101_64x4d_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine resnext101_64x4d_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ No newline at end of file diff --git a/models/cv/classification/resnext101_64x4d/igie/scripts/infer_resnext101_64x4d_fp16_performance.sh b/models/cv/classification/resnext101_64x4d/igie/scripts/infer_resnext101_64x4d_fp16_performance.sh new file mode 100644 index 00000000..3ccf3bc5 --- /dev/null +++ b/models/cv/classification/resnext101_64x4d/igie/scripts/infer_resnext101_64x4d_fp16_performance.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="resnext101_64x4d.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path resnext101_64x4d_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine resnext101_64x4d_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ + --perf_only True \ No newline at end of file -- Gitee From 67e3b05a41c8939a839776cf4ad3df7b1e08b76c Mon Sep 17 00:00:00 2001 From: YoungPeng Date: Wed, 9 Oct 2024 15:35:26 +0800 Subject: [PATCH 08/10] Add: shufflenetv2_x1_5 inference script. --- .../shufflenetv2_x1_5/igie/README.md | 47 +++++ .../shufflenetv2_x1_5/igie/build_engine.py | 73 +++++++ .../shufflenetv2_x1_5/igie/export.py | 61 ++++++ .../shufflenetv2_x1_5/igie/inference.py | 186 ++++++++++++++++++ .../infer_shufflenetv2_x1_5_fp16_accuracy.sh | 35 ++++ ...nfer_shufflenetv2_x1_5_fp16_performance.sh | 36 ++++ 6 files changed, 438 insertions(+) create mode 100644 models/cv/classification/shufflenetv2_x1_5/igie/README.md create mode 100644 models/cv/classification/shufflenetv2_x1_5/igie/build_engine.py create mode 100644 models/cv/classification/shufflenetv2_x1_5/igie/export.py create mode 100644 models/cv/classification/shufflenetv2_x1_5/igie/inference.py create mode 100644 models/cv/classification/shufflenetv2_x1_5/igie/scripts/infer_shufflenetv2_x1_5_fp16_accuracy.sh create mode 100644 models/cv/classification/shufflenetv2_x1_5/igie/scripts/infer_shufflenetv2_x1_5_fp16_performance.sh diff --git a/models/cv/classification/shufflenetv2_x1_5/igie/README.md b/models/cv/classification/shufflenetv2_x1_5/igie/README.md new file mode 100644 index 00000000..5bd87eab --- /dev/null +++ b/models/cv/classification/shufflenetv2_x1_5/igie/README.md @@ -0,0 +1,47 @@ +# ShuffleNetV2_x1_5 + +## Description + +ShuffleNetV2_x1_5 is a lightweight convolutional neural network specifically designed for efficient image recognition tasks on resource-constrained devices. It achieves high performance and low latency through the introduction of channel shuffling and pointwise group convolutions. Despite its small model size, it offers high accuracy and is suitable for a variety of vision tasks in mobile devices and embedded systems. + +## Setup + +### Install + +```bash +pip3 install onnx +pip3 install tqdm +``` + +### Download + +Pretrained model: + +Dataset: to download the validation dataset. + +### Model Conversion + +```bash +python3 export.py --weight shufflenetv2_x1_5-3c479a10.pth --output shufflenetv2_x1_5.onnx +``` + +## Inference + +```bash +export DATASETS_DIR=/Path/to/imagenet_val/ +``` + +### FP16 + +```bash +# Accuracy +bash scripts/infer_shufflenetv2_x1_5_fp16_accuracy.sh +# Performance +bash scripts/infer_shufflenetv2_x1_5_fp16_performance.sh +``` + +## Results + +| Model | BatchSize | Precision | FPS | Top-1(%) | Top-5(%) | +| ----------------- | --------- | --------- | -------- | -------- | -------- | +| ShuffleNetV2_x1_5 | 32 | FP16 | 7478.728 | 72.755 | 91.031 | diff --git a/models/cv/classification/shufflenetv2_x1_5/igie/build_engine.py b/models/cv/classification/shufflenetv2_x1_5/igie/build_engine.py new file mode 100644 index 00000000..d3626ae7 --- /dev/null +++ b/models/cv/classification/shufflenetv2_x1_5/igie/build_engine.py @@ -0,0 +1,73 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import tvm +import argparse +from tvm import relay +from tvm.relay.import_model import import_model_to_igie + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--model_path", + type=str, + required=True, + help="original model path.") + + parser.add_argument("--engine_path", + type=str, + required=True, + help="igie export engine path.") + + parser.add_argument("--input", + type=str, + required=True, + help=""" + input info of the model, format should be: + input_name:input_shape + eg: --input input:1,3,224,224. + """) + + parser.add_argument("--precision", + type=str, + choices=["fp32", "fp16", "int8"], + required=True, + help="model inference precision.") + + args = parser.parse_args() + + return args + +def main(): + args = parse_args() + + # get input valueinfo + input_name, input_shape = args.input.split(":") + shape = tuple([int(s) for s in input_shape.split(",")]) + input_dict = {input_name: shape} + + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + + mod, params = import_model_to_igie(args.model_path, input_dict, backend="igie") + + # build engine + lib = tvm.relay.build(mod, target=target, params=params, precision=args.precision) + + # export engine + lib.export_library(args.engine_path) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/shufflenetv2_x1_5/igie/export.py b/models/cv/classification/shufflenetv2_x1_5/igie/export.py new file mode 100644 index 00000000..0f89ba7c --- /dev/null +++ b/models/cv/classification/shufflenetv2_x1_5/igie/export.py @@ -0,0 +1,61 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import torch +import torchvision +import argparse + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--weight", + type=str, + required=True, + help="pytorch model weight.") + + parser.add_argument("--output", + type=str, + required=True, + help="export onnx model path.") + + args = parser.parse_args() + return args + +def main(): + args = parse_args() + + model = torchvision.models.shufflenet_v2_x1_5() + model.load_state_dict(torch.load(args.weight)) + model.eval() + + input_names = ['input'] + output_names = ['output'] + dynamic_axes = {'input': {0: '-1'}, 'output': {0: '-1'}} + dummy_input = torch.randn(1, 3, 224, 224) + + torch.onnx.export( + model, + dummy_input, + args.output, + input_names = input_names, + dynamic_axes = dynamic_axes, + output_names = output_names, + opset_version=13 + ) + + print("Export onnx model successfully! ") + +if __name__ == "__main__": + main() diff --git a/models/cv/classification/shufflenetv2_x1_5/igie/inference.py b/models/cv/classification/shufflenetv2_x1_5/igie/inference.py new file mode 100644 index 00000000..3aef3ec7 --- /dev/null +++ b/models/cv/classification/shufflenetv2_x1_5/igie/inference.py @@ -0,0 +1,186 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +import argparse +import tvm +import torch +import torchvision +import numpy as np +from tvm import relay +from tqdm import tqdm +from torchvision import transforms +from torchvision.transforms.functional import InterpolationMode + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--engine", + type=str, + required=True, + help="igie engine path.") + + parser.add_argument("--batchsize", + type=int, + required=True, + help="inference batch size.") + + parser.add_argument("--datasets", + type=str, + required=True, + help="datasets path.") + + parser.add_argument("--input_name", + type=str, + required=True, + help="input name of the model.") + + parser.add_argument("--warmup", + type=int, + default=3, + help="number of warmup before test.") + + parser.add_argument("--num_workers", + type=int, + default=16, + help="number of workers used in pytorch dataloader.") + + parser.add_argument("--acc_target", + type=float, + default=None, + help="Model inference Accuracy target.") + + parser.add_argument("--fps_target", + type=float, + default=None, + help="Model inference FPS target.") + + parser.add_argument("--perf_only", + type=bool, + default=False, + help="Run performance test only") + + args = parser.parse_args() + + return args + +def get_dataloader(data_path, batch_size, num_workers): + dataset = torchvision.datasets.ImageFolder( + data_path, + transforms.Compose( + [ + transforms.Resize(256, interpolation=InterpolationMode.BILINEAR), + transforms.CenterCrop(224), + transforms.PILToTensor(), + transforms.ConvertImageDtype(torch.float), + transforms.Normalize( + mean=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225) + ) + ] + ) + ) + + dataloader = torch.utils.data.DataLoader(dataset, batch_size, num_workers=num_workers) + + return dataloader + +def get_topk_accuracy(pred, label): + if isinstance(pred, np.ndarray): + pred = torch.from_numpy(pred) + + if isinstance(label, np.ndarray): + label = torch.from_numpy(label) + + top1_acc = 0 + top5_acc = 0 + for idx in range(len(label)): + label_value = label[idx] + if label_value == torch.topk(pred[idx].float(), 1).indices.data: + top1_acc += 1 + top5_acc += 1 + + elif label_value in torch.topk(pred[idx].float(), 5).indices.data: + top5_acc += 1 + + return top1_acc, top5_acc + +def main(): + args = parse_args() + + batch_size = args.batchsize + + # create iluvatar target & device + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + device = tvm.device(target.kind.name, 0) + + # load engine + lib = tvm.runtime.load_module(args.engine) + + # create runtime from engine + module = tvm.contrib.graph_executor.GraphModule(lib["default"](device)) + + # just run perf test + if args.perf_only: + ftimer = module.module.time_evaluator("run", device, number=100, repeat=1) + prof_res = np.array(ftimer().results) * 1000 + fps = batch_size * 1000 / np.mean(prof_res) + print(f"\n* Mean inference time: {np.mean(prof_res):.3f} ms, Mean fps: {fps:.3f}") + else: + # warm up + for _ in range(args.warmup): + module.run() + + # get dataloader + dataloader = get_dataloader(args.datasets, batch_size, args.num_workers) + + top1_acc = 0 + top5_acc = 0 + total_num = 0 + + for image, label in tqdm(dataloader): + + # pad the last batch + pad_batch = len(image) != batch_size + + if pad_batch: + origin_size = len(image) + image = np.resize(image, (batch_size, *image.shape[1:])) + + module.set_input(args.input_name, tvm.nd.array(image, device)) + + # run inference + module.run() + + pred = module.get_output(0).asnumpy() + + if pad_batch: + pred = pred[:origin_size] + + # get batch accuracy + batch_top1_acc, batch_top5_acc = get_topk_accuracy(pred, label) + + top1_acc += batch_top1_acc + top5_acc += batch_top5_acc + total_num += batch_size + + result_stat = {} + result_stat["acc@1"] = round(top1_acc / total_num * 100.0, 3) + result_stat["acc@5"] = round(top5_acc / total_num * 100.0, 3) + + print(f"\n* Top1 acc: {result_stat['acc@1']} %, Top5 acc: {result_stat['acc@5']} %") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/shufflenetv2_x1_5/igie/scripts/infer_shufflenetv2_x1_5_fp16_accuracy.sh b/models/cv/classification/shufflenetv2_x1_5/igie/scripts/infer_shufflenetv2_x1_5_fp16_accuracy.sh new file mode 100644 index 00000000..9be9264c --- /dev/null +++ b/models/cv/classification/shufflenetv2_x1_5/igie/scripts/infer_shufflenetv2_x1_5_fp16_accuracy.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="shufflenetv2_x1_5.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path shufflenetv2_x1_5_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine shufflenetv2_x1_5_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ No newline at end of file diff --git a/models/cv/classification/shufflenetv2_x1_5/igie/scripts/infer_shufflenetv2_x1_5_fp16_performance.sh b/models/cv/classification/shufflenetv2_x1_5/igie/scripts/infer_shufflenetv2_x1_5_fp16_performance.sh new file mode 100644 index 00000000..cc5a424d --- /dev/null +++ b/models/cv/classification/shufflenetv2_x1_5/igie/scripts/infer_shufflenetv2_x1_5_fp16_performance.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="shufflenetv2_x1_5.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path shufflenetv2_x1_5_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine shufflenetv2_x1_5_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ + --perf_only True \ No newline at end of file -- Gitee From 865c38337bc7be7043a8d3c649661db5a19b08b5 Mon Sep 17 00:00:00 2001 From: YoungPeng Date: Thu, 10 Oct 2024 16:48:26 +0800 Subject: [PATCH 09/10] Add: mvitv2_base inference script. --- .../classification/mvitv2_base/igie/README.md | 68 +++++++ .../mvitv2_base/igie/build_engine.py | 73 +++++++ .../classification/mvitv2_base/igie/export.py | 74 +++++++ .../mvitv2_base/igie/inference.py | 185 ++++++++++++++++++ .../infer_mvitv2_base_fp16_accuracy.sh | 35 ++++ .../infer_mvitv2_base_fp16_performance.sh | 36 ++++ 6 files changed, 471 insertions(+) create mode 100644 models/cv/classification/mvitv2_base/igie/README.md create mode 100644 models/cv/classification/mvitv2_base/igie/build_engine.py create mode 100644 models/cv/classification/mvitv2_base/igie/export.py create mode 100644 models/cv/classification/mvitv2_base/igie/inference.py create mode 100644 models/cv/classification/mvitv2_base/igie/scripts/infer_mvitv2_base_fp16_accuracy.sh create mode 100644 models/cv/classification/mvitv2_base/igie/scripts/infer_mvitv2_base_fp16_performance.sh diff --git a/models/cv/classification/mvitv2_base/igie/README.md b/models/cv/classification/mvitv2_base/igie/README.md new file mode 100644 index 00000000..199a9597 --- /dev/null +++ b/models/cv/classification/mvitv2_base/igie/README.md @@ -0,0 +1,68 @@ +# MViTv2-base + +## Description + +MViTv2_base is an efficient multi-scale vision Transformer model designed specifically for image classification tasks. By employing a multi-scale structure and hierarchical representation, it effectively captures both global and local image features while maintaining computational efficiency. The MViTv2_base has demonstrated excellent performance on multiple standard datasets and is suitable for a variety of visual recognition tasks. + +## Setup + +### Install + +```bash +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-dev + +pip3 install onnx +pip3 install tqdm +pip3 install onnxsim +pip3 install mmcv==1.5.3 +pip3 install mmcls +``` + +### Download + +Pretrained model: + +Dataset: to download the validation dataset. + +### Model Conversion + +```bash +# git clone mmpretrain +git clone -b v0.24.0 https://github.com/open-mmlab/mmpretrain.git + +# export onnx model +python3 export.py --cfg mmpretrain/configs/mvit/mvitv2-base_8xb256_in1k.py --weight mvitv2-base_3rdparty_in1k_20220722-9c4f0a17.pth --output mvitv2_base.onnx + +# Use onnxsim optimize onnx model +onnxsim mvitv2_base.onnx mvitv2_base_opt.onnx + +``` + +## Inference + +```bash +export DATASETS_DIR=/Path/to/imagenet_val/ +``` + +### FP16 + +```bash +# Accuracy +bash scripts/infer_mvitv2_base_fp16_accuracy.sh +# Performance +bash scripts/infer_mvitv2_base_fp16_performance.sh +``` + +## Results + +| Model | BatchSize | Precision | FPS | Top-1(%) | Top-5(%) | +| ----------- | --------- | --------- | -------- | -------- | -------- | +| MViTv2-base | 16 | FP16 | 58.76 | 84.226 | 96.848 | + +## Reference + +MViTv2-base: diff --git a/models/cv/classification/mvitv2_base/igie/build_engine.py b/models/cv/classification/mvitv2_base/igie/build_engine.py new file mode 100644 index 00000000..d3626ae7 --- /dev/null +++ b/models/cv/classification/mvitv2_base/igie/build_engine.py @@ -0,0 +1,73 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import tvm +import argparse +from tvm import relay +from tvm.relay.import_model import import_model_to_igie + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--model_path", + type=str, + required=True, + help="original model path.") + + parser.add_argument("--engine_path", + type=str, + required=True, + help="igie export engine path.") + + parser.add_argument("--input", + type=str, + required=True, + help=""" + input info of the model, format should be: + input_name:input_shape + eg: --input input:1,3,224,224. + """) + + parser.add_argument("--precision", + type=str, + choices=["fp32", "fp16", "int8"], + required=True, + help="model inference precision.") + + args = parser.parse_args() + + return args + +def main(): + args = parse_args() + + # get input valueinfo + input_name, input_shape = args.input.split(":") + shape = tuple([int(s) for s in input_shape.split(",")]) + input_dict = {input_name: shape} + + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + + mod, params = import_model_to_igie(args.model_path, input_dict, backend="igie") + + # build engine + lib = tvm.relay.build(mod, target=target, params=params, precision=args.precision) + + # export engine + lib.export_library(args.engine_path) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/mvitv2_base/igie/export.py b/models/cv/classification/mvitv2_base/igie/export.py new file mode 100644 index 00000000..d78b898b --- /dev/null +++ b/models/cv/classification/mvitv2_base/igie/export.py @@ -0,0 +1,74 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import argparse + +import torch +from mmcls.apis import init_model + +class Model(torch.nn.Module): + def __init__(self, config_file, checkpoint_file): + super().__init__() + self.model = init_model(config_file, checkpoint_file, device="cpu") + + def forward(self, x): + head = self.model.simple_test(x) + return head + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--weight", + type=str, + required=True, + help="pytorch model weight.") + + parser.add_argument("--cfg", + type=str, + required=True, + help="model config file.") + + parser.add_argument("--output", + type=str, + required=True, + help="export onnx model path.") + + args = parser.parse_args() + return args + +def main(): + args = parse_args() + + config_file = args.cfg + checkpoint_file = args.weight + model = Model(config_file, checkpoint_file).eval() + + input_names = ['input'] + output_names = ['output'] + dummy_input = torch.randn(16, 3, 224, 224) + + torch.onnx.export( + model, + dummy_input, + args.output, + input_names = input_names, + output_names = output_names, + opset_version=13 + ) + + print("Export onnx model successfully! ") + +if __name__ == '__main__': + main() + diff --git a/models/cv/classification/mvitv2_base/igie/inference.py b/models/cv/classification/mvitv2_base/igie/inference.py new file mode 100644 index 00000000..c42cf871 --- /dev/null +++ b/models/cv/classification/mvitv2_base/igie/inference.py @@ -0,0 +1,185 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +import argparse +import tvm +import torch +import torchvision +import numpy as np +from tvm import relay +from tqdm import tqdm +from torchvision import transforms + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--engine", + type=str, + required=True, + help="igie engine path.") + + parser.add_argument("--batchsize", + type=int, + required=True, + help="inference batch size.") + + parser.add_argument("--datasets", + type=str, + required=True, + help="datasets path.") + + parser.add_argument("--input_name", + type=str, + required=True, + help="input name of the model.") + + parser.add_argument("--warmup", + type=int, + default=3, + help="number of warmup before test.") + + parser.add_argument("--num_workers", + type=int, + default=16, + help="number of workers used in pytorch dataloader.") + + parser.add_argument("--acc_target", + type=float, + default=None, + help="Model inference Accuracy target.") + + parser.add_argument("--fps_target", + type=float, + default=None, + help="Model inference FPS target.") + + parser.add_argument("--perf_only", + type=bool, + default=False, + help="Run performance test only") + + args = parser.parse_args() + + return args + +def get_dataloader(data_path, batch_size, num_workers): + dataset = torchvision.datasets.ImageFolder( + data_path, + transforms.Compose( + [ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.PILToTensor(), + transforms.ConvertImageDtype(torch.float), + transforms.Normalize( + mean=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225) + ) + ] + ) + ) + + dataloader = torch.utils.data.DataLoader(dataset, batch_size, num_workers=num_workers) + + return dataloader + +def get_topk_accuracy(pred, label): + if isinstance(pred, np.ndarray): + pred = torch.from_numpy(pred) + + if isinstance(label, np.ndarray): + label = torch.from_numpy(label) + + top1_acc = 0 + top5_acc = 0 + for idx in range(len(label)): + label_value = label[idx] + if label_value == torch.topk(pred[idx].float(), 1).indices.data: + top1_acc += 1 + top5_acc += 1 + + elif label_value in torch.topk(pred[idx].float(), 5).indices.data: + top5_acc += 1 + + return top1_acc, top5_acc + +def main(): + args = parse_args() + + batch_size = args.batchsize + + # create iluvatar target & device + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + device = tvm.device(target.kind.name, 0) + + # load engine + lib = tvm.runtime.load_module(args.engine) + + # create runtime from engine + module = tvm.contrib.graph_executor.GraphModule(lib["default"](device)) + + # just run perf test + if args.perf_only: + ftimer = module.module.time_evaluator("run", device, number=100, repeat=1) + prof_res = np.array(ftimer().results) * 1000 + fps = batch_size * 1000 / np.mean(prof_res) + print(f"\n* Mean inference time: {np.mean(prof_res):.3f} ms, Mean fps: {fps:.3f}") + else: + # warm up + for _ in range(args.warmup): + module.run() + + # get dataloader + dataloader = get_dataloader(args.datasets, batch_size, args.num_workers) + + top1_acc = 0 + top5_acc = 0 + total_num = 0 + + for image, label in tqdm(dataloader): + + # pad the last batch + pad_batch = len(image) != batch_size + + if pad_batch: + origin_size = len(image) + image = np.resize(image, (batch_size, *image.shape[1:])) + + module.set_input(args.input_name, tvm.nd.array(image, device)) + + # run inference + module.run() + + pred = module.get_output(0).asnumpy() + + if pad_batch: + pred = pred[:origin_size] + + # get batch accuracy + batch_top1_acc, batch_top5_acc = get_topk_accuracy(pred, label) + + top1_acc += batch_top1_acc + top5_acc += batch_top5_acc + total_num += batch_size + + result_stat = {} + result_stat["acc@1"] = round(top1_acc / total_num * 100.0, 3) + result_stat["acc@5"] = round(top5_acc / total_num * 100.0, 3) + + print(f"\n* Top1 acc: {result_stat['acc@1']} %, Top5 acc: {result_stat['acc@5']} %") + +if __name__ == "__main__": + main() diff --git a/models/cv/classification/mvitv2_base/igie/scripts/infer_mvitv2_base_fp16_accuracy.sh b/models/cv/classification/mvitv2_base/igie/scripts/infer_mvitv2_base_fp16_accuracy.sh new file mode 100644 index 00000000..85f66d50 --- /dev/null +++ b/models/cv/classification/mvitv2_base/igie/scripts/infer_mvitv2_base_fp16_accuracy.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=16 +model_path="mvitv2_base_opt.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path mvitv2_base_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine mvitv2_base_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} diff --git a/models/cv/classification/mvitv2_base/igie/scripts/infer_mvitv2_base_fp16_performance.sh b/models/cv/classification/mvitv2_base/igie/scripts/infer_mvitv2_base_fp16_performance.sh new file mode 100644 index 00000000..d54dac50 --- /dev/null +++ b/models/cv/classification/mvitv2_base/igie/scripts/infer_mvitv2_base_fp16_performance.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=16 +model_path="mvitv2_base_opt.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path mvitv2_base_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine mvitv2_base_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ + --perf_only True \ No newline at end of file -- Gitee From 2f249448a4652e6b9ed0631d86bc63547a97dd90 Mon Sep 17 00:00:00 2001 From: YoungPeng Date: Thu, 10 Oct 2024 17:34:57 +0800 Subject: [PATCH 10/10] Add: resnetv1d50 inference script. --- .../classification/resnetv1d50/igie/README.md | 64 ++++++ .../resnetv1d50/igie/build_engine.py | 73 +++++++ .../classification/resnetv1d50/igie/export.py | 76 +++++++ .../resnetv1d50/igie/inference.py | 185 ++++++++++++++++++ .../infer_resnetv1d50_fp16_accuracy.sh | 35 ++++ .../infer_resnetv1d50_fp16_performance.sh | 36 ++++ 6 files changed, 469 insertions(+) create mode 100644 models/cv/classification/resnetv1d50/igie/README.md create mode 100644 models/cv/classification/resnetv1d50/igie/build_engine.py create mode 100644 models/cv/classification/resnetv1d50/igie/export.py create mode 100644 models/cv/classification/resnetv1d50/igie/inference.py create mode 100644 models/cv/classification/resnetv1d50/igie/scripts/infer_resnetv1d50_fp16_accuracy.sh create mode 100644 models/cv/classification/resnetv1d50/igie/scripts/infer_resnetv1d50_fp16_performance.sh diff --git a/models/cv/classification/resnetv1d50/igie/README.md b/models/cv/classification/resnetv1d50/igie/README.md new file mode 100644 index 00000000..0609b34f --- /dev/null +++ b/models/cv/classification/resnetv1d50/igie/README.md @@ -0,0 +1,64 @@ +# ResNetV1D-50 + +## Description + +ResNetV1D-50 is an enhanced version of ResNetV1-50 that incorporates changes like dilated convolutions and adjusted downsampling, leading to better performance in large-scale image classification tasks. Its ability to capture richer image features makes it a popular choice in deep learning models. + +## Setup + +### Install + +```bash +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-dev + +pip3 install onnx +pip3 install tqdm +pip3 install mmcv==1.5.3 +pip3 install mmcls +``` + +### Download + +Pretrained model: + +Dataset: to download the validation dataset. + +### Model Conversion + +```bash +# git clone mmpretrain +git clone -b v0.24.0 https://github.com/open-mmlab/mmpretrain.git + +# export onnx model +python3 export.py --cfg mmpretrain/configs/resnet/resnetv1d50_b32x8_imagenet.py --weight resnetv1d50_b32x8_imagenet_20210531-db14775a.pth --output resnetv1d50.onnx + +``` + +## Inference + +```bash +export DATASETS_DIR=/Path/to/imagenet_val/ +``` + +### FP16 + +```bash +# Accuracy +bash scripts/infer_resnetv1d50_fp16_accuracy.sh +# Performance +bash scripts/infer_resnetv1d50_fp16_performance.sh +``` + +## Results + +| Model | BatchSize | Precision | FPS | Top-1(%) | Top-5(%) | +| ------------ | --------- | --------- | -------- | -------- | -------- | +| ResNetV1D-50 | 32 | FP16 | 4017.92 | 77.517 | 93.538 | + +## Reference + +ResNetV1D-50: diff --git a/models/cv/classification/resnetv1d50/igie/build_engine.py b/models/cv/classification/resnetv1d50/igie/build_engine.py new file mode 100644 index 00000000..d3626ae7 --- /dev/null +++ b/models/cv/classification/resnetv1d50/igie/build_engine.py @@ -0,0 +1,73 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import tvm +import argparse +from tvm import relay +from tvm.relay.import_model import import_model_to_igie + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--model_path", + type=str, + required=True, + help="original model path.") + + parser.add_argument("--engine_path", + type=str, + required=True, + help="igie export engine path.") + + parser.add_argument("--input", + type=str, + required=True, + help=""" + input info of the model, format should be: + input_name:input_shape + eg: --input input:1,3,224,224. + """) + + parser.add_argument("--precision", + type=str, + choices=["fp32", "fp16", "int8"], + required=True, + help="model inference precision.") + + args = parser.parse_args() + + return args + +def main(): + args = parse_args() + + # get input valueinfo + input_name, input_shape = args.input.split(":") + shape = tuple([int(s) for s in input_shape.split(",")]) + input_dict = {input_name: shape} + + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + + mod, params = import_model_to_igie(args.model_path, input_dict, backend="igie") + + # build engine + lib = tvm.relay.build(mod, target=target, params=params, precision=args.precision) + + # export engine + lib.export_library(args.engine_path) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/models/cv/classification/resnetv1d50/igie/export.py b/models/cv/classification/resnetv1d50/igie/export.py new file mode 100644 index 00000000..84045099 --- /dev/null +++ b/models/cv/classification/resnetv1d50/igie/export.py @@ -0,0 +1,76 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import argparse + +import torch +from mmcls.apis import init_model + +class Model(torch.nn.Module): + def __init__(self, config_file, checkpoint_file): + super().__init__() + self.model = init_model(config_file, checkpoint_file, device="cpu") + + def forward(self, x): + head = self.model.simple_test(x) + return head + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--weight", + type=str, + required=True, + help="pytorch model weight.") + + parser.add_argument("--cfg", + type=str, + required=True, + help="model config file.") + + parser.add_argument("--output", + type=str, + required=True, + help="export onnx model path.") + + args = parser.parse_args() + return args + +def main(): + args = parse_args() + + config_file = args.cfg + checkpoint_file = args.weight + model = Model(config_file, checkpoint_file).eval() + + input_names = ['input'] + output_names = ['output'] + dynamic_axes = {'input': {0: '-1'}, 'output': {0: '-1'}} + dummy_input = torch.randn(1, 3, 224, 224) + + torch.onnx.export( + model, + dummy_input, + args.output, + input_names = input_names, + dynamic_axes = dynamic_axes, + output_names = output_names, + opset_version=13 + ) + + print("Export onnx model successfully! ") + +if __name__ == '__main__': + main() + diff --git a/models/cv/classification/resnetv1d50/igie/inference.py b/models/cv/classification/resnetv1d50/igie/inference.py new file mode 100644 index 00000000..c42cf871 --- /dev/null +++ b/models/cv/classification/resnetv1d50/igie/inference.py @@ -0,0 +1,185 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +import argparse +import tvm +import torch +import torchvision +import numpy as np +from tvm import relay +from tqdm import tqdm +from torchvision import transforms + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--engine", + type=str, + required=True, + help="igie engine path.") + + parser.add_argument("--batchsize", + type=int, + required=True, + help="inference batch size.") + + parser.add_argument("--datasets", + type=str, + required=True, + help="datasets path.") + + parser.add_argument("--input_name", + type=str, + required=True, + help="input name of the model.") + + parser.add_argument("--warmup", + type=int, + default=3, + help="number of warmup before test.") + + parser.add_argument("--num_workers", + type=int, + default=16, + help="number of workers used in pytorch dataloader.") + + parser.add_argument("--acc_target", + type=float, + default=None, + help="Model inference Accuracy target.") + + parser.add_argument("--fps_target", + type=float, + default=None, + help="Model inference FPS target.") + + parser.add_argument("--perf_only", + type=bool, + default=False, + help="Run performance test only") + + args = parser.parse_args() + + return args + +def get_dataloader(data_path, batch_size, num_workers): + dataset = torchvision.datasets.ImageFolder( + data_path, + transforms.Compose( + [ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.PILToTensor(), + transforms.ConvertImageDtype(torch.float), + transforms.Normalize( + mean=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225) + ) + ] + ) + ) + + dataloader = torch.utils.data.DataLoader(dataset, batch_size, num_workers=num_workers) + + return dataloader + +def get_topk_accuracy(pred, label): + if isinstance(pred, np.ndarray): + pred = torch.from_numpy(pred) + + if isinstance(label, np.ndarray): + label = torch.from_numpy(label) + + top1_acc = 0 + top5_acc = 0 + for idx in range(len(label)): + label_value = label[idx] + if label_value == torch.topk(pred[idx].float(), 1).indices.data: + top1_acc += 1 + top5_acc += 1 + + elif label_value in torch.topk(pred[idx].float(), 5).indices.data: + top5_acc += 1 + + return top1_acc, top5_acc + +def main(): + args = parse_args() + + batch_size = args.batchsize + + # create iluvatar target & device + target = tvm.target.iluvatar(model="MR", options="-libs=cudnn,cublas,ixinfer") + device = tvm.device(target.kind.name, 0) + + # load engine + lib = tvm.runtime.load_module(args.engine) + + # create runtime from engine + module = tvm.contrib.graph_executor.GraphModule(lib["default"](device)) + + # just run perf test + if args.perf_only: + ftimer = module.module.time_evaluator("run", device, number=100, repeat=1) + prof_res = np.array(ftimer().results) * 1000 + fps = batch_size * 1000 / np.mean(prof_res) + print(f"\n* Mean inference time: {np.mean(prof_res):.3f} ms, Mean fps: {fps:.3f}") + else: + # warm up + for _ in range(args.warmup): + module.run() + + # get dataloader + dataloader = get_dataloader(args.datasets, batch_size, args.num_workers) + + top1_acc = 0 + top5_acc = 0 + total_num = 0 + + for image, label in tqdm(dataloader): + + # pad the last batch + pad_batch = len(image) != batch_size + + if pad_batch: + origin_size = len(image) + image = np.resize(image, (batch_size, *image.shape[1:])) + + module.set_input(args.input_name, tvm.nd.array(image, device)) + + # run inference + module.run() + + pred = module.get_output(0).asnumpy() + + if pad_batch: + pred = pred[:origin_size] + + # get batch accuracy + batch_top1_acc, batch_top5_acc = get_topk_accuracy(pred, label) + + top1_acc += batch_top1_acc + top5_acc += batch_top5_acc + total_num += batch_size + + result_stat = {} + result_stat["acc@1"] = round(top1_acc / total_num * 100.0, 3) + result_stat["acc@5"] = round(top5_acc / total_num * 100.0, 3) + + print(f"\n* Top1 acc: {result_stat['acc@1']} %, Top5 acc: {result_stat['acc@5']} %") + +if __name__ == "__main__": + main() diff --git a/models/cv/classification/resnetv1d50/igie/scripts/infer_resnetv1d50_fp16_accuracy.sh b/models/cv/classification/resnetv1d50/igie/scripts/infer_resnetv1d50_fp16_accuracy.sh new file mode 100644 index 00000000..c3a3d869 --- /dev/null +++ b/models/cv/classification/resnetv1d50/igie/scripts/infer_resnetv1d50_fp16_accuracy.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="resnetv1d50.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path resnetv1d50_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine resnetv1d50_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} diff --git a/models/cv/classification/resnetv1d50/igie/scripts/infer_resnetv1d50_fp16_performance.sh b/models/cv/classification/resnetv1d50/igie/scripts/infer_resnetv1d50_fp16_performance.sh new file mode 100644 index 00000000..194d2900 --- /dev/null +++ b/models/cv/classification/resnetv1d50/igie/scripts/infer_resnetv1d50_fp16_performance.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +batchsize=32 +model_path="resnetv1d50.onnx" +datasets_path=${DATASETS_DIR} + +# build engine +python3 build_engine.py \ + --model_path ${model_path} \ + --input input:${batchsize},3,224,224 \ + --precision fp16 \ + --engine_path resnetv1d50_bs_${batchsize}_fp16.so + + +# inference +python3 inference.py \ + --engine resnetv1d50_bs_${batchsize}_fp16.so \ + --batchsize ${batchsize} \ + --input_name input \ + --datasets ${datasets_path} \ + --perf_only True \ No newline at end of file -- Gitee