From 2b228b3ce1f7b02713069bf4f546dd0062dd0431 Mon Sep 17 00:00:00 2001 From: "feihu.fang" Date: Mon, 21 Jul 2025 15:39:07 +0800 Subject: [PATCH] modify ixrt clip --- models/cv/classification/clip/ixrt/README.md | 13 ++++++---- .../cv/classification/clip/ixrt/inference.py | 22 ++++++++++------ .../ixrt/scripts/infer_clip_fp16_accuracy.sh | 26 +++++++++---------- .../scripts/infer_clip_fp16_performance.sh | 26 +++++++++---------- .../ixrt_common/config/CLIP_CONFIG | 6 +++++ 5 files changed, 52 insertions(+), 41 deletions(-) create mode 100644 models/cv/classification/ixrt_common/config/CLIP_CONFIG diff --git a/models/cv/classification/clip/ixrt/README.md b/models/cv/classification/clip/ixrt/README.md index cb888f2d..4eb88cb7 100644 --- a/models/cv/classification/clip/ixrt/README.md +++ b/models/cv/classification/clip/ixrt/README.md @@ -33,15 +33,18 @@ pip3 install -r requirements.txt ```bash python3 export.py --output clip.onnx - -# Use onnxsim optimize onnx model -onnxsim clip.onnx clip_sim.onnx ``` ## Model Inference ```bash -export DATASETS_DIR=/Path/to/imagenet_val/ +git clone https://gitee.com/deep-spark/iluvatar-corex-ixrt.git --depth=1 +export OPTIMIER_FILE=./iluvatar-corex-ixrt/tools/optimizer/optimizer.py +export DATASETS_DIR=/path/to/imagenet_val/ +export PROJ_DIR=./ +export CHECKPOINTS_DIR=./ +export RUN_DIR=./ +export CONFIG_DIR=../../ixrt_common/config/CLIP_CONFIG ``` ### FP16 @@ -57,4 +60,4 @@ bash scripts/infer_clip_fp16_performance.sh | Model | BatchSize | Precision | FPS | Top-1(%) | Top-5(%) | | :----: | :----: | :----: | :----: | :----: | :----: | -| CLIP | 32 | FP16 | 496.91 | 59.68 | 86.16 | +| CLIP | 32 | FP16 | 350.94 | 59.68 | 86.14 | diff --git a/models/cv/classification/clip/ixrt/inference.py b/models/cv/classification/clip/ixrt/inference.py index d6a3a756..a7a372b3 100644 --- a/models/cv/classification/clip/ixrt/inference.py +++ b/models/cv/classification/clip/ixrt/inference.py @@ -78,13 +78,19 @@ class ModelRunner: self.context.execute_v2(self.allocations) class ClassificationRunner(ModelRunner): - def load_input(self, batch_data): + def load_input(self, input_id, image, attention): if self.is_ixrt_backend(): - err, = cuda.cuMemcpyHtoD(self.inputs[0]["allocation"], batch_data, batch_data.nbytes) + err, = cuda.cuMemcpyHtoD(self.inputs[0]["allocation"], input_id, input_id.nbytes) + assert(err == cuda.CUresult.CUDA_SUCCESS) + err, = cuda.cuMemcpyHtoD(self.inputs[1]["allocation"], image, image.nbytes) + assert(err == cuda.CUresult.CUDA_SUCCESS) + err, = cuda.cuMemcpyHtoD(self.inputs[2]["allocation"], attention, attention.nbytes) assert(err == cuda.CUresult.CUDA_SUCCESS) elif self.is_ort_backend(): - self.inputs[0]["allocation"] = batch_data + self.inputs[0]["allocation"] = input_id + self.inputs[1]["allocation"] = image + self.inputs[2]["allocation"] = attention else: raise @@ -153,12 +159,12 @@ def main(config): # for input_id, image, attention, label in tqdm(dataloader): with tqdm(total= len(dataloader)) as _tqdm: - for idx, (batch_data, batch_label) in enumerate(dataloader): - batch_data = batch_data.numpy().astype(runner.inputs[0]["dtype"]) - batch_data = np.ascontiguousarray(batch_data) - total_sample += batch_data.shape[0] + for idx, (input_id, image, attention, batch_label) in enumerate(dataloader): + image = image.astype(runner.inputs[1]["dtype"]) + image = np.ascontiguousarray(image) + total_sample += image.shape[0] - runner.load_input(batch_data) + runner.load_input(input_id, image, attention) runner.run() output = runner.fetch_output()[0] diff --git a/models/cv/classification/clip/ixrt/scripts/infer_clip_fp16_accuracy.sh b/models/cv/classification/clip/ixrt/scripts/infer_clip_fp16_accuracy.sh index 2a3b2b8d..c5601bf2 100644 --- a/models/cv/classification/clip/ixrt/scripts/infer_clip_fp16_accuracy.sh +++ b/models/cv/classification/clip/ixrt/scripts/infer_clip_fp16_accuracy.sh @@ -54,31 +54,29 @@ echo Model Name : ${MODEL_NAME} echo Onnx Path : ${ORIGINE_MODEL} step=0 -SIM_MODEL=${CHECKPOINTS_DIR}/${MODEL_NAME}_sim.onnx -# Simplify Model +# Change Input size and Simplify Model let step++ echo; -echo [STEP ${step}] : Simplify Model -if [ -f ${SIM_MODEL} ];then - echo " "Simplify Model, ${SIM_MODEL} has been existed +echo [STEP ${step}] : Change Batchsize and Simplify +SIM_MODEL=${CHECKPOINTS_DIR}/${MODEL_NAME}_${BSZ}_sim.onnx +if [ -f $SIM_MODEL ];then + echo " "Change Input size and Simplify Model Skip, $SIM_MODEL has been existed else - python3 ${RUN_DIR}/simplify_model.py \ - --origin_model $ORIGINE_MODEL \ - --output_model ${SIM_MODEL} + onnxsim ${ORIGINE_MODEL} ${SIM_MODEL} \ + --overwrite-input-shape input_ids:1000,22 pixel_values:${BSZ},3,224,224 attention_mask:1000,22 echo " "Generate ${SIM_MODEL} fi -# Change Batchsize +# optimizer let step++ echo; -echo [STEP ${step}] : Change Batchsize -FINAL_MODEL=${CHECKPOINTS_DIR}/${MODEL_NAME}_${BSZ}.onnx +echo [STEP ${step}] : Optimizer +FINAL_MODEL=${CHECKPOINTS_DIR}/${MODEL_NAME}_${BSZ}_sim_end.onnx if [ -f $FINAL_MODEL ];then - echo " "Change Batchsize Skip, $FINAL_MODEL has been existed + echo " "Optimizer Model Skip, $FINAL_MODEL has been existed else - python3 ${RUN_DIR}/modify_batchsize.py --batch_size ${BSZ} \ - --origin_model ${SIM_MODEL} --output_model ${FINAL_MODEL} + python3 ${OPTIMIER_FILE} --onnx ${SIM_MODEL} --model_type vit echo " "Generate ${FINAL_MODEL} fi diff --git a/models/cv/classification/clip/ixrt/scripts/infer_clip_fp16_performance.sh b/models/cv/classification/clip/ixrt/scripts/infer_clip_fp16_performance.sh index e7a4f1a7..3f137912 100644 --- a/models/cv/classification/clip/ixrt/scripts/infer_clip_fp16_performance.sh +++ b/models/cv/classification/clip/ixrt/scripts/infer_clip_fp16_performance.sh @@ -54,31 +54,29 @@ echo Model Name : ${MODEL_NAME} echo Onnx Path : ${ORIGINE_MODEL} step=0 -SIM_MODEL=${CHECKPOINTS_DIR}/${MODEL_NAME}_sim.onnx -# Simplify Model +# Change Input size and Simplify Model let step++ echo; -echo [STEP ${step}] : Simplify Model -if [ -f ${SIM_MODEL} ];then - echo " "Simplify Model, ${SIM_MODEL} has been existed +echo [STEP ${step}] : Change Batchsize and Simplify +SIM_MODEL=${CHECKPOINTS_DIR}/${MODEL_NAME}_${BSZ}_sim.onnx +if [ -f $SIM_MODEL ];then + echo " "Change Input size and Simplify Model Skip, $SIM_MODEL has been existed else - python3 ${RUN_DIR}/simplify_model.py \ - --origin_model $ORIGINE_MODEL \ - --output_model ${SIM_MODEL} + onnxsim ${ORIGINE_MODEL} ${SIM_MODEL} \ + --overwrite-input-shape input_ids:1000,22 pixel_values:${BSZ},3,224,224 attention_mask:1000,22 echo " "Generate ${SIM_MODEL} fi -# Change Batchsize +# optimizer let step++ echo; -echo [STEP ${step}] : Change Batchsize -FINAL_MODEL=${CHECKPOINTS_DIR}/${MODEL_NAME}_${BSZ}.onnx +echo [STEP ${step}] : Optimizer +FINAL_MODEL=${CHECKPOINTS_DIR}/${MODEL_NAME}_${BSZ}_sim_end.onnx if [ -f $FINAL_MODEL ];then - echo " "Change Batchsize Skip, $FINAL_MODEL has been existed + echo " "Optimizer Model Skip, $FINAL_MODEL has been existed else - python3 ${RUN_DIR}/modify_batchsize.py --batch_size ${BSZ} \ - --origin_model ${SIM_MODEL} --output_model ${FINAL_MODEL} + python3 ${OPTIMIER_FILE} --onnx ${SIM_MODEL} --model_type vit echo " "Generate ${FINAL_MODEL} fi diff --git a/models/cv/classification/ixrt_common/config/CLIP_CONFIG b/models/cv/classification/ixrt_common/config/CLIP_CONFIG new file mode 100644 index 00000000..0aa1670b --- /dev/null +++ b/models/cv/classification/ixrt_common/config/CLIP_CONFIG @@ -0,0 +1,6 @@ +# IMGSIZE : 模型输入hw大小 +# MODEL_NAME : 生成onnx/engine的basename +# ORIGINE_MODEL : 原始onnx文件名称 +IMGSIZE=224 +MODEL_NAME=Clip +ORIGINE_MODEL=clip.onnx \ No newline at end of file -- Gitee