From 98ec6974dbd9d3e268f360353b1bad974bb9c068 Mon Sep 17 00:00:00 2001 From: "hongliang.yuan" Date: Fri, 28 Feb 2025 15:02:24 +0800 Subject: [PATCH] add cv/classification and detect ci --- cv/classification/acmix/pytorch/ci/prepare.sh | 20 ++ cv/classification/acnet/pytorch/ci/prepare.sh | 30 ++ .../alexnet/tensorflow/README.md | 2 +- cv/classification/cbam/pytorch/ci/prepare.sh | 19 ++ .../convnext/pytorch/ci/prepare.sh | 34 ++ .../densenet/paddlepaddle/README.md | 17 +- .../efficientnet_b0/paddlepaddle/README.md | 12 +- .../fasternet/pytorch/ci/prepare.sh | 27 ++ .../googlenet/paddlepaddle/README.md | 12 +- .../internimage/pytorch/ci/prepare.sh | 54 ++++ .../mobilenetv3/paddlepaddle/README.md | 13 +- .../paddlepaddle/README.md | 16 +- .../pp-lcnet/paddlepaddle/README.md | 8 +- .../repmlp/pytorch/ci/prepare.sh | 28 ++ .../repvgg/pytorch/ci/prepare.sh | 30 ++ .../repvit/pytorch/ci/prepare.sh | 26 ++ .../res2net50_14w_8s/paddlepaddle/README.md | 11 +- .../resnest50/paddlepaddle/README.md | 12 +- .../resnet50/paddlepaddle/README.md | 15 +- .../resnet50/tensorflow/README.md | 3 + .../run_train_resnet50_imagenette.sh | 2 +- .../run_train_resnet50_multigpu_imagenette.sh | 2 +- .../se_resnet50_vd/paddlepaddle/README.md | 15 +- .../paddlepaddle/requirements.txt | 13 - .../shufflenetv2/paddlepaddle/README.md | 11 +- .../swin_transformer/paddlepaddle/README.md | 13 +- .../swin_transformer/pytorch/ci/prepare.sh | 27 ++ cv/classification/vgg/paddlepaddle/README.md | 10 +- .../tensorflow/run_train_vgg16_imagenet.sh | 2 +- .../wavemlp/pytorch/ci/prepare.sh | 31 ++ .../xception/paddlepaddle/README.md | 12 +- cv/detection/centermask2/pytorch/README.md | 2 +- .../centermask2/pytorch/ci/prepare.sh | 45 +++ cv/detection/centernet/paddlepaddle/README.md | 11 +- cv/detection/centernet/pytorch/README.md | 2 +- cv/detection/centernet/pytorch/ci/prepare.sh | 46 +++ cv/detection/co-detr/pytorch/README.md | 19 +- cv/detection/co-detr/pytorch/requirements.txt | 6 - .../cornernet_mmdet/pytorch/README.md | 5 +- cv/detection/dcnv2_mmdet/pytorch/README.md | 5 +- cv/detection/detr/paddlepaddle/README.md | 12 +- cv/detection/fasterrcnn/pytorch/ci/prepare.sh | 39 +++ cv/detection/fcos/paddlepaddle/README.md | 12 +- cv/detection/fcos/pytorch/ci/prepare.sh | 34 ++ cv/detection/mamba_yolo/pytorch/README.md | 3 +- cv/detection/mamba_yolo/pytorch/ci/prepare.sh | 46 +++ cv/detection/maskrcnn/paddlepaddle/README.md | 13 + cv/detection/maskrcnn/pytorch/ci/prepare.sh | 44 +++ cv/detection/oc_sort/paddlepaddle/README.md | 8 +- cv/detection/picodet/paddlepaddle/README.md | 8 +- cv/detection/pp-yoloe/paddlepaddle/README.md | 10 +- cv/detection/pp_yoloe+/paddlepaddle/README.md | 8 + cv/detection/pvanet/pytorch/ci/prepare.sh | 35 ++ cv/detection/pvanet/pytorch/coco_eval.py | 2 +- .../reppoints_mmdet/pytorch/README.md | 5 +- cv/detection/retinanet/paddlepaddle/README.md | 8 +- cv/detection/retinanet/pytorch/ci/prepare.sh | 33 ++ cv/detection/rt-detr/pytorch/ci/prepare.sh | 43 +++ cv/detection/solov2/paddlepaddle/README.md | 12 +- cv/detection/ssd/paddlepaddle/README.md | 10 +- cv/detection/ssd/pytorch/base/prepare.py | 23 +- cv/detection/ssd/pytorch/ci/prepare.sh | 35 ++ cv/detection/ssd/tensorflow/README.md | 7 + cv/detection/ssd/tensorflow/ci/prepare.sh | 35 ++ cv/detection/yolov3/paddlepaddle/README.md | 14 +- cv/detection/yolov3/tensorflow/README.md | 6 + cv/detection/yolov3/tensorflow/ci/prepare.sh | 30 ++ cv/detection/yolov3/tensorflow/train_fast.py | 2 + cv/detection/yolov5/paddlepaddle/README.md | 11 +- cv/detection/yolov5/pytorch/ci/prepare.sh | 70 ++++ tests/run_test.py | 300 ++++++++++++++++++ 71 files changed, 1469 insertions(+), 117 deletions(-) create mode 100644 cv/classification/acmix/pytorch/ci/prepare.sh create mode 100644 cv/classification/acnet/pytorch/ci/prepare.sh create mode 100644 cv/classification/cbam/pytorch/ci/prepare.sh create mode 100644 cv/classification/convnext/pytorch/ci/prepare.sh create mode 100644 cv/classification/fasternet/pytorch/ci/prepare.sh create mode 100644 cv/classification/internimage/pytorch/ci/prepare.sh create mode 100644 cv/classification/repmlp/pytorch/ci/prepare.sh create mode 100644 cv/classification/repvgg/pytorch/ci/prepare.sh create mode 100644 cv/classification/repvit/pytorch/ci/prepare.sh delete mode 100644 cv/classification/se_resnet50_vd/paddlepaddle/requirements.txt create mode 100644 cv/classification/swin_transformer/pytorch/ci/prepare.sh create mode 100644 cv/classification/wavemlp/pytorch/ci/prepare.sh create mode 100644 cv/detection/centermask2/pytorch/ci/prepare.sh create mode 100644 cv/detection/centernet/pytorch/ci/prepare.sh delete mode 100644 cv/detection/co-detr/pytorch/requirements.txt create mode 100644 cv/detection/fasterrcnn/pytorch/ci/prepare.sh create mode 100644 cv/detection/fcos/pytorch/ci/prepare.sh create mode 100644 cv/detection/mamba_yolo/pytorch/ci/prepare.sh create mode 100644 cv/detection/maskrcnn/pytorch/ci/prepare.sh create mode 100644 cv/detection/pvanet/pytorch/ci/prepare.sh create mode 100644 cv/detection/retinanet/pytorch/ci/prepare.sh create mode 100644 cv/detection/rt-detr/pytorch/ci/prepare.sh create mode 100644 cv/detection/ssd/pytorch/ci/prepare.sh create mode 100644 cv/detection/ssd/tensorflow/ci/prepare.sh create mode 100644 cv/detection/yolov3/tensorflow/ci/prepare.sh create mode 100644 cv/detection/yolov5/pytorch/ci/prepare.sh create mode 100644 tests/run_test.py diff --git a/cv/classification/acmix/pytorch/ci/prepare.sh b/cv/classification/acmix/pytorch/ci/prepare.sh new file mode 100644 index 00000000..b7988b53 --- /dev/null +++ b/cv/classification/acmix/pytorch/ci/prepare.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +pip install termcolor==1.1.0 yacs==0.1.8 timm==0.4.5 +timeout 1800 bash run.sh 4 acmix_swin_small_patch4_window7_224.yaml ../imagenet \ No newline at end of file diff --git a/cv/classification/acnet/pytorch/ci/prepare.sh b/cv/classification/acnet/pytorch/ci/prepare.sh new file mode 100644 index 00000000..7154b51c --- /dev/null +++ b/cv/classification/acnet/pytorch/ci/prepare.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +git clone https://github.com/DingXiaoH/ACNet.git +cd ACNet +ln -s /mnt/deepspark/data/datasets/imagenet ./imagenet_data +rm -rf acnet/acb.py +rm -rf utils/misc.py +mv ../acb.py acnet/ +mv ../misc.py utils/ +export PYTHONPATH=$PYTHONPATH:. +# export CUDA_VISIBLE_DEVICES=0 +timeout 1800 python3 -m torch.distributed.launch --nproc_per_node=1 acnet/do_acnet.py -a sres18 -b acb --local_rank 0 +# export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +# timeout 1800 python3 -m torch.distributed.launch --nproc_per_node=8 acnet/do_acnet.py -a sres18 -b acb \ No newline at end of file diff --git a/cv/classification/alexnet/tensorflow/README.md b/cv/classification/alexnet/tensorflow/README.md index f95aad1c..59a8cb47 100644 --- a/cv/classification/alexnet/tensorflow/README.md +++ b/cv/classification/alexnet/tensorflow/README.md @@ -57,4 +57,4 @@ bash run_train_alexnet_multigpu_imagenet.sh ## References -- [TensorFlow Models](https://github.com/tensorflow/models) +- [TensorFlow/benchmarks](https://github.com/tensorflow/benchmarks/tree/master/scripts/tf_cnn_benchmarks) diff --git a/cv/classification/cbam/pytorch/ci/prepare.sh b/cv/classification/cbam/pytorch/ci/prepare.sh new file mode 100644 index 00000000..463f5330 --- /dev/null +++ b/cv/classification/cbam/pytorch/ci/prepare.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +timeout 1800 python3 train_imagenet.py --ngpu 4 --workers 20 --arch resnet --depth 50 --epochs 100 --batch-size 256 --lr 0.1 --att-type CBAM --prefix RESNET50_IMAGENET_CBAM ./imagenet \ No newline at end of file diff --git a/cv/classification/convnext/pytorch/ci/prepare.sh b/cv/classification/convnext/pytorch/ci/prepare.sh new file mode 100644 index 00000000..a4e79944 --- /dev/null +++ b/cv/classification/convnext/pytorch/ci/prepare.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +pip install timm==0.4.12 tensorboardX six torch torchvision + +git clone https://github.com/facebookresearch/ConvNeXt.git +cd ConvNeXt +git checkout 048efcea897d999aed302f2639b6270aedf8d4c8 +# fix torch._six import error +sed -i 's/from torch._six import inf/from torch import inf/g' utils.py +timeout 1800 python3 main.py --model convnext_tiny \ + --drop_path 0.1 \ + --batch_size 128 \ + --lr 4e-3 \ + --update_freq 4 \ + --model_ema true \ + --model_ema_eval true \ + --data_path ../imagenet \ + --output_dir ../save_results \ No newline at end of file diff --git a/cv/classification/densenet/paddlepaddle/README.md b/cv/classification/densenet/paddlepaddle/README.md index 678b4e19..df4f0002 100644 --- a/cv/classification/densenet/paddlepaddle/README.md +++ b/cv/classification/densenet/paddlepaddle/README.md @@ -34,16 +34,15 @@ imagenet ### Install Dependencies ```bash -git clone --recursive https://github.com/PaddlePaddle/PaddleClas.git - -cd PaddleClas/ - -yum install mesa-libGL -y - +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 +cd PaddleClas pip3 install -r requirements.txt -pip3 install protobuf==3.20.3 -pip3 install urllib3==1.26.13 - python3 setup.py install ``` diff --git a/cv/classification/efficientnet_b0/paddlepaddle/README.md b/cv/classification/efficientnet_b0/paddlepaddle/README.md index af8c0c9a..28416892 100644 --- a/cv/classification/efficientnet_b0/paddlepaddle/README.md +++ b/cv/classification/efficientnet_b0/paddlepaddle/README.md @@ -47,14 +47,16 @@ sed -i 's#^#val/#g' val_list.txt ### Install Dependencies ```bash +# Install libGL +## CentOS yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx -git clone -b release/2.5 https://github.com/PaddlePaddle/PaddleClas.git -cd PaddleClas/ +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 +cd PaddleClas pip3 install -r requirements.txt -pip3 install paddleclas -pip3 install protobuf==3.20.3 -pip3 install urllib3==1.26.15 +python3 setup.py install ``` diff --git a/cv/classification/fasternet/pytorch/ci/prepare.sh b/cv/classification/fasternet/pytorch/ci/prepare.sh new file mode 100644 index 00000000..88585352 --- /dev/null +++ b/cv/classification/fasternet/pytorch/ci/prepare.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +pip install -r requirements.txt +git clone https://github.com/JierunChen/FasterNet.git +cd FasterNet +git checkout e8fba4465ae912359c9f661a72b14e39347e4954 +timeout 1800 python3 train_test.py -g 0,1,2,3 --num_nodes 1 -n 4 -b 4096 -e 2000 \ + --data_dir ../imagenet \ + --pin_memory --wandb_project_name fasternet \ + --model_ckpt_dir ./model_ckpt/$(date +'%Y%m%d_%H%M%S') \ + --cfg cfg/fasternet_t0.yaml \ No newline at end of file diff --git a/cv/classification/googlenet/paddlepaddle/README.md b/cv/classification/googlenet/paddlepaddle/README.md index 6f4139d9..61b77aba 100644 --- a/cv/classification/googlenet/paddlepaddle/README.md +++ b/cv/classification/googlenet/paddlepaddle/README.md @@ -34,9 +34,16 @@ imagenet ### Install Dependencies ```bash -git clone --recursive https://github.com/PaddlePaddle/PaddleClas.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 cd PaddleClas pip3 install -r requirements.txt +python3 setup.py install ``` ## Model Training @@ -61,3 +68,6 @@ export FLAGS_cudnn_batchnorm_spatial_persistent=True export CUDA_VISIBLE_DEVICES=0,1,2,3 python3 -u -m paddle.distributed.launch --gpus=0,1,2,3 tools/train.py -c ppcls/configs/ImageNet/Inception/GoogLeNet.yaml ``` + +## Reference +- [PaddleClas](https://github.com/PaddlePaddle/PaddleClas) \ No newline at end of file diff --git a/cv/classification/internimage/pytorch/ci/prepare.sh b/cv/classification/internimage/pytorch/ci/prepare.sh new file mode 100644 index 00000000..592cac8b --- /dev/null +++ b/cv/classification/internimage/pytorch/ci/prepare.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +## Install mmcv +cd mmcv/ +bash clean_mmcv.sh +bash build_mmcv.sh +bash install_mmcv.sh +cd ../ + +## Install timm and mmdet +pip3 install timm==0.6.11 mmdet==2.28.1 + +pip3 install addict yapf opencv-python termcolor yacs pyyaml scipy + +cd ./ops_dcnv3 +sh ./make.sh +# unit test (should see all checking is True) +# python3 test.py +cd ../ +# Training on 8 GPUs +# export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +# export LOCAL_SIZE=8 +# python3 -m torch.distributed.launch --nproc_per_node 8 --master_port 12345 main.py --cfg configs/internimage_t_1k_224.yaml --data-path ./imagenet + +# Training on 1 GPU +export CUDA_VISIBLE_DEVICES=0 +export LOCAL_SIZE=1 +timeout 1800 python3 main.py --cfg configs/internimage_t_1k_224.yaml --data-path ./imagenet \ No newline at end of file diff --git a/cv/classification/mobilenetv3/paddlepaddle/README.md b/cv/classification/mobilenetv3/paddlepaddle/README.md index 1b707203..a0dd9fba 100644 --- a/cv/classification/mobilenetv3/paddlepaddle/README.md +++ b/cv/classification/mobilenetv3/paddlepaddle/README.md @@ -35,9 +35,18 @@ imagenet ### Install Dependencies ```bash -git clone https://github.com/PaddlePaddle/PaddleClas.git -cd PaddleClas/ +## Step 1: Installing +```bash +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 +cd PaddleClas pip3 install -r requirements.txt +python3 setup.py install ``` ## Model Training diff --git a/cv/classification/mobilenetv3_large_x1_0/paddlepaddle/README.md b/cv/classification/mobilenetv3_large_x1_0/paddlepaddle/README.md index 996ddf9f..f76bf582 100644 --- a/cv/classification/mobilenetv3_large_x1_0/paddlepaddle/README.md +++ b/cv/classification/mobilenetv3_large_x1_0/paddlepaddle/README.md @@ -35,15 +35,17 @@ imagenet ### Install Dependencies ```bash -yum install mesa-libGL -y - -git clone https://github.com/PaddlePaddle/PaddleClas.git -cd PaddleClas/ +```bash +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 +cd PaddleClas pip3 install -r requirements.txt python3 setup.py install - -pip3 install protobuf==3.20.3 -pip3 install urllib3==1.26.13 ``` ## Model Training diff --git a/cv/classification/pp-lcnet/paddlepaddle/README.md b/cv/classification/pp-lcnet/paddlepaddle/README.md index 54decf1f..04f93785 100644 --- a/cv/classification/pp-lcnet/paddlepaddle/README.md +++ b/cv/classification/pp-lcnet/paddlepaddle/README.md @@ -35,7 +35,13 @@ imagenet ### Install Dependencies ```bash -git clone --recursive https://github.com/PaddlePaddle/PaddleClas.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 cd PaddleClas pip3 install -r requirements.txt python3 setup.py install diff --git a/cv/classification/repmlp/pytorch/ci/prepare.sh b/cv/classification/repmlp/pytorch/ci/prepare.sh new file mode 100644 index 00000000..81b99280 --- /dev/null +++ b/cv/classification/repmlp/pytorch/ci/prepare.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +pip3 install timm yacs +git clone https://github.com/DingXiaoH/RepMLP.git +cd RepMLP +git checkout 3eff13fa0257af28663880d870f327d665f0a8e2 +# fix --local-rank for torch 2.x +sed -i 's/--local_rank/--local-rank/g' main_repmlp.py +# change dataset load +sed -i "s@dataset = torchvision.datasets.ImageNet(root=config.DATA.DATA_PATH, split='train' if is_train else 'val', transform=transform)@dataset = datasets.ImageFolder(os.path.join(config.DATA.DATA_PATH, prefix), transform=transform)@" data/build.py + +timeout 1800 python3 -m torch.distributed.launch --nproc_per_node 4 --master_port 12349 main_repmlp.py --arch RepMLPNet-B256 --batch-size 32 --tag my_experiment --opts TRAIN.EPOCHS 100 TRAIN.BASE_LR 0.001 TRAIN.WEIGHT_DECAY 0.1 TRAIN.OPTIMIZER.NAME adamw TRAIN.OPTIMIZER.MOMENTUM 0.9 TRAIN.WARMUP_LR 5e-7 TRAIN.MIN_LR 0.0 TRAIN.WARMUP_EPOCHS 10 AUG.PRESET raug15 AUG.MIXUP 0.4 AUG.CUTMIX 1.0 DATA.IMG_SIZE 256 --data-path ../imagenet \ No newline at end of file diff --git a/cv/classification/repvgg/pytorch/ci/prepare.sh b/cv/classification/repvgg/pytorch/ci/prepare.sh new file mode 100644 index 00000000..b97a6abe --- /dev/null +++ b/cv/classification/repvgg/pytorch/ci/prepare.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +pip3 install timm yacs +git clone https://github.com/DingXiaoH/RepVGG.git +cd RepVGG +git checkout eae7c5204001eaf195bbe2ee72fb6a37855cce33 +# fix --local-rank for torch 2.x +sed -i 's/--local_rank/--local-rank/g' main.py + +# change dataset load +# Tips: "import os" into data/build.py +sed -i "s@dataset = torchvision.datasets.ImageNet(root=config.DATA.DATA_PATH, split='train' if is_train else 'val', transform=transform)@dataset = datasets.ImageFolder(os.path.join(config.DATA.DATA_PATH, prefix), transform=transform)@" data/build.py + +timeout 1800 python3 -m torch.distributed.launch --nproc_per_node 4 --master_port 12349 main.py --arch RepVGG-A0 --data-path ../imagenet --batch-size 32 --tag train_from_scratch --output ./ --opts TRAIN.EPOCHS 300 TRAIN.BASE_LR 0.1 TRAIN.WEIGHT_DECAY 1e-4 TRAIN.WARMUP_EPOCHS 5 MODEL.LABEL_SMOOTHING 0.1 AUG.PRESET weak AUG.MIXUP 0.0 DATA.DATASET imagenet DATA.IMG_SIZE 224 \ No newline at end of file diff --git a/cv/classification/repvit/pytorch/ci/prepare.sh b/cv/classification/repvit/pytorch/ci/prepare.sh new file mode 100644 index 00000000..6b5940a8 --- /dev/null +++ b/cv/classification/repvit/pytorch/ci/prepare.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +git clone https://github.com/THU-MIG/RepViT.git +cd RepViT +git checkout 298f42075eda5d2e6102559fad260c970769d34e +pip3 install -r requirements.txt +# On single GPU +timeout 1800 python3 main.py --model repvit_m0_9 --data-path ../imagenet --dist-eval + +# # Multiple GPUs on one machine +# python3 -m torch.distributed.launch --nproc_per_node=8 --master_port 12346 --use_env main.py --model repvit_m0_9 --data-path ./imagenet --dist-eval \ No newline at end of file diff --git a/cv/classification/res2net50_14w_8s/paddlepaddle/README.md b/cv/classification/res2net50_14w_8s/paddlepaddle/README.md index c34af020..c19b2c64 100644 --- a/cv/classification/res2net50_14w_8s/paddlepaddle/README.md +++ b/cv/classification/res2net50_14w_8s/paddlepaddle/README.md @@ -34,11 +34,16 @@ imagenet ### Install Dependencies ```bash -git clone -b release/2.5 https://github.com/PaddlePaddle/PaddleClas.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 cd PaddleClas pip3 install -r requirements.txt -pip3 install protobuf==3.20.3 urllib3==1.26.6 -yum install -y mesa-libGL +python3 setup.py install ``` ## Model Training diff --git a/cv/classification/resnest50/paddlepaddle/README.md b/cv/classification/resnest50/paddlepaddle/README.md index 9717d0ca..0fae57ee 100644 --- a/cv/classification/resnest50/paddlepaddle/README.md +++ b/cv/classification/resnest50/paddlepaddle/README.md @@ -35,9 +35,16 @@ imagenet ### Install Dependencies ```bash -git clone --recursive https://github.com/PaddlePaddle/PaddleClas.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 cd PaddleClas pip3 install -r requirements.txt +python3 setup.py install ``` ## Model Training @@ -55,3 +62,6 @@ python3 -u -m paddle.distributed.launch --gpus=0,1,2,3 tools/train.py -c ppcls/c | Model | GPU | FP32 | |-----------|------------|--------------| | ResNeSt50 | BI-V100 x4 | Acc@1=0.7677 | + +## Reference +- [PaddleClas](https://github.com/PaddlePaddle/PaddleClas) \ No newline at end of file diff --git a/cv/classification/resnet50/paddlepaddle/README.md b/cv/classification/resnet50/paddlepaddle/README.md index 5151cfb8..5e029d80 100644 --- a/cv/classification/resnet50/paddlepaddle/README.md +++ b/cv/classification/resnet50/paddlepaddle/README.md @@ -34,12 +34,16 @@ imagenet ### Install Dependencies ```bash -git clone --recursive https://github.com/PaddlePaddle/PaddleClas.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 cd PaddleClas pip3 install -r requirements.txt -yum install mesa-libGL -y -pip3 install urllib3==1.26.6 -pip3 install protobuf==3.20.3 +python3 setup.py install ``` Tips: for `PaddleClas` training, the images path in train_list.txt and val_list.txt must contain `train/` and `val/` directories: @@ -71,3 +75,6 @@ python3 -u -m paddle.distributed.launch --gpus=0,1,2,3 tools/train.py -c ppcls/c | Model | GPU | FP32 | |----------|------------|------------------------------------| | ResNet50 | BI-V100 x4 | Acc@1=76.27,FPS=80.37,BatchSize=64 | + +## Reference +- [PaddleClas](https://github.com/PaddlePaddle/PaddleClas) \ No newline at end of file diff --git a/cv/classification/resnet50/tensorflow/README.md b/cv/classification/resnet50/tensorflow/README.md index c12d7e5e..3832bc54 100644 --- a/cv/classification/resnet50/tensorflow/README.md +++ b/cv/classification/resnet50/tensorflow/README.md @@ -39,3 +39,6 @@ bash run_train_resnet50_multigpu_imagenette.sh | Model | GPU | acc | fps | |----------|------------|--------|-------| | ResNet50 | BI-V100 x8 | 0.9860 | 236.9 | + +## Reference +- [TensorFlow/benchmarks](https://github.com/tensorflow/benchmarks/tree/master/scripts/tf_cnn_benchmarks) \ No newline at end of file diff --git a/cv/classification/resnet50/tensorflow/run_train_resnet50_imagenette.sh b/cv/classification/resnet50/tensorflow/run_train_resnet50_imagenette.sh index 171fd876..a8a17884 100644 --- a/cv/classification/resnet50/tensorflow/run_train_resnet50_imagenette.sh +++ b/cv/classification/resnet50/tensorflow/run_train_resnet50_imagenette.sh @@ -40,7 +40,7 @@ do done python3 -u tf_cnn_benchmarks.py\ - --data_name=imagenette --data_dir=${DATA_DIR}\ + --data_name=imagenet --data_dir=${DATA_DIR}\ --data_format=NCHW --batch_size=${BATCH_SIZE}\ --model=resnet50 --optimizer=${OPTIMIZER} --num_gpus=1\ --weight_decay=1e-4 --train_dir=${TRAIN_DIR}\ diff --git a/cv/classification/resnet50/tensorflow/run_train_resnet50_multigpu_imagenette.sh b/cv/classification/resnet50/tensorflow/run_train_resnet50_multigpu_imagenette.sh index 1a2c33ad..c4ab33fa 100644 --- a/cv/classification/resnet50/tensorflow/run_train_resnet50_multigpu_imagenette.sh +++ b/cv/classification/resnet50/tensorflow/run_train_resnet50_multigpu_imagenette.sh @@ -42,7 +42,7 @@ done source ./get_num_devices.sh UMD_WAITAFTERLAUNCH=1 python3 -u tf_cnn_benchmarks.py\ - --data_name=imagenette --data_dir=${DATA_DIR}\ + --data_name=imagenet --data_dir=${DATA_DIR}\ --data_format=NCHW --batch_size=${BATCH_SIZE}\ --model=resnet50 --optimizer=${OPTIMIZER} --num_gpus=${IX_NUM_CUDA_VISIBLE_DEVICES}\ --weight_decay=1e-4 --train_dir=${TRAIN_DIR}\ diff --git a/cv/classification/se_resnet50_vd/paddlepaddle/README.md b/cv/classification/se_resnet50_vd/paddlepaddle/README.md index 9f74f52c..88ae5742 100644 --- a/cv/classification/se_resnet50_vd/paddlepaddle/README.md +++ b/cv/classification/se_resnet50_vd/paddlepaddle/README.md @@ -46,17 +46,24 @@ sed -i 's#^#val/#g' val_list.txt ### Install Dependencies ```bash -pip3 install -r requirements.txt -python3 -m pip install urllib3==1.26.6 +# Install libGL +## CentOS yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx -git clone -b release/2.5 https://github.com/PaddlePaddle/PaddleClas.git +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 +cd PaddleClas +pip3 install -r requirements.txt +python3 setup.py install ``` ## Model Training ```bash cd PaddleClas/ +# Link your dataset to the default location +ln -s /path/to/imagenet ./dataset/ILSVRC2012 export CUDA_VISIBLE_DEVICES=0,1,2,3 python3 -m paddle.distributed.launch --gpus="0,1,2,3" tools/train.py -c ./ppcls/configs/ImageNet/SENet/SE_ResNet50_vd.yaml ``` @@ -69,4 +76,4 @@ python3 -m paddle.distributed.launch --gpus="0,1,2,3" tools/train.py -c ./ppcls/ ## References -- [PaddleClas](https://github.com/PaddlePaddle/PaddleClas/tree/release/2.5) +- [PaddleClas](https://github.com/PaddlePaddle/PaddleClas) diff --git a/cv/classification/se_resnet50_vd/paddlepaddle/requirements.txt b/cv/classification/se_resnet50_vd/paddlepaddle/requirements.txt deleted file mode 100644 index 4d1ad019..00000000 --- a/cv/classification/se_resnet50_vd/paddlepaddle/requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ -prettytable -ujson -opencv-python==4.6.0.66 -pillow -tqdm -PyYAML>=5.1 -visualdl>=2.2.0 -scipy>=1.0.0 -scikit-learn==1.0.2 -gast==0.3.3 -faiss-cpu -easydict -visualdl==2.4.0 \ No newline at end of file diff --git a/cv/classification/shufflenetv2/paddlepaddle/README.md b/cv/classification/shufflenetv2/paddlepaddle/README.md index 65bdbbf9..00d02403 100644 --- a/cv/classification/shufflenetv2/paddlepaddle/README.md +++ b/cv/classification/shufflenetv2/paddlepaddle/README.md @@ -34,16 +34,17 @@ imagenet ### Install Dependencies ```bash +# Install libGL +## CentOS yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx -git clone --recursive https://github.com/PaddlePaddle/PaddleClas.git -cd PaddleClas/ +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 +cd PaddleClas pip3 install -r requirements.txt python3 setup.py install -pip3 install protobuf==3.20.3 -pip3 install urllib3==1.26.13 - ``` ## Model Training diff --git a/cv/classification/swin_transformer/paddlepaddle/README.md b/cv/classification/swin_transformer/paddlepaddle/README.md index d63622ab..dda047a8 100644 --- a/cv/classification/swin_transformer/paddlepaddle/README.md +++ b/cv/classification/swin_transformer/paddlepaddle/README.md @@ -35,9 +35,16 @@ imagenet ### Install Dependencies ```bash -git clone --recursive https://github.com/PaddlePaddle/PaddleClas.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 cd PaddleClas pip3 install -r requirements.txt +python3 setup.py install ``` ## Model Training @@ -57,3 +64,7 @@ python3 -u -m paddle.distributed.launch --gpus=0,1,2,3,4,5,6,7 tools/train.py -c | Model | GPU | FP32 | |------------------|------------|--------------| | Swin Transformer | BI-V100 x8 | Acc@1=0.8024 | + +## References + +- [PaddleClas](https://github.com/PaddlePaddle/PaddleClas) \ No newline at end of file diff --git a/cv/classification/swin_transformer/pytorch/ci/prepare.sh b/cv/classification/swin_transformer/pytorch/ci/prepare.sh new file mode 100644 index 00000000..330fd94a --- /dev/null +++ b/cv/classification/swin_transformer/pytorch/ci/prepare.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +git clone https://github.com/microsoft/Swin-Transformer.git +git checkout f82860bfb5225915aca09c3227159ee9e1df874d +cd Swin-Transformer +pip install timm==0.4.12 yacs +## fix --local-rank for torch 2.x +sed -i 's/--local_rank/--local-rank/g' main.py + +timeout 1800 python3 -m torch.distributed.launch --nproc_per_node 4 --master_port 12345 main.py \ + --cfg configs/swin/swin_tiny_patch4_window7_224.yaml --data-path ../imagenet --batch-size 128 \ No newline at end of file diff --git a/cv/classification/vgg/paddlepaddle/README.md b/cv/classification/vgg/paddlepaddle/README.md index df26ef19..44829679 100644 --- a/cv/classification/vgg/paddlepaddle/README.md +++ b/cv/classification/vgg/paddlepaddle/README.md @@ -34,12 +34,16 @@ imagenet ### Install Dependencies ```bash -git clone https://github.com/PaddlePaddle/PaddleClas.git -``` +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx -```bash +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 cd PaddleClas pip3 install -r requirements.txt +python3 setup.py install ``` ## Model Training diff --git a/cv/classification/vgg/tensorflow/run_train_vgg16_imagenet.sh b/cv/classification/vgg/tensorflow/run_train_vgg16_imagenet.sh index 342a88f6..aece8bfe 100644 --- a/cv/classification/vgg/tensorflow/run_train_vgg16_imagenet.sh +++ b/cv/classification/vgg/tensorflow/run_train_vgg16_imagenet.sh @@ -17,7 +17,7 @@ export TF_CUDNN_USE_AUTOTUNE=1 export TF_CPP_MIN_LOG_LEVEL=1 -: ${BATCH_SIZE:=256} +: ${BATCH_SIZE:=32} #TRAIN_EPOCHS=10 # optional optimizer: adam, rmsprop, momentum, sgd OPTIMIZER=momentum diff --git a/cv/classification/wavemlp/pytorch/ci/prepare.sh b/cv/classification/wavemlp/pytorch/ci/prepare.sh new file mode 100644 index 00000000..79daef5f --- /dev/null +++ b/cv/classification/wavemlp/pytorch/ci/prepare.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +pip install thop timm==0.4.5 torchprofile +git clone https://github.com/huawei-noah/Efficient-AI-Backbones.git +cd Efficient-AI-Backbones/wavemlp_pytorch/ +git checkout 25531f7fdcf61e300b47c52ba80973d0af8bb011 + +# fix --local-rank for torch 2.x +sed -i 's/--local_rank/--local-rank/g' train.py +# change dataset +sed -i "s@from timm.data import Dataset@from timm.data import ImageDataset@" train.py +sed -i "s@dataset_train = Dataset(train_dir)@dataset_train = ImageDataset(train_dir)@" train.py +sed -i "s@dataset_eval = Dataset(eval_dir)@dataset_eval = ImageDataset(eval_dir)@" train.py +sed -i 's/args.max_history/100/g' train.py + +python3 -m torch.distributed.launch --nproc_per_node 8 --nnodes=1 --node_rank=0 train.py ../imagenet/ --output ./output/ --model WaveMLP_T_dw --sched cosine --epochs 300 --opt adamw -j 8 --warmup-lr 1e-6 --mixup .8 --cutmix 1.0 --model-ema --model-ema-decay 0.99996 --aa rand-m9-mstd0.5-inc1 --color-jitter 0.4 --warmup-epochs 5 --opt-eps 1e-8 --repeated-aug --remode pixel --reprob 0.25 --amp --lr 1e-3 --weight-decay .05 --drop 0 --drop-path 0.1 -b 128 \ No newline at end of file diff --git a/cv/classification/xception/paddlepaddle/README.md b/cv/classification/xception/paddlepaddle/README.md index 425e1078..c31dd9ef 100644 --- a/cv/classification/xception/paddlepaddle/README.md +++ b/cv/classification/xception/paddlepaddle/README.md @@ -47,10 +47,16 @@ sed -i 's#^#val/#g' val_list.txt ### Install Dependencies ```bash -git clone -b release/2.5 https://github.com/PaddlePaddle/PaddleClas.git -cd PaddleClas -pip3 install scikit-learn easydict visualdl==2.2.0 urllib3==1.26.6 +# Install libGL +## CentOS yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 +cd PaddleClas +pip3 install -r requirements.txt +python3 setup.py install ``` ## Model Training diff --git a/cv/detection/centermask2/pytorch/README.md b/cv/detection/centermask2/pytorch/README.md index 6992d069..4d5d8e3c 100644 --- a/cv/detection/centermask2/pytorch/README.md +++ b/cv/detection/centermask2/pytorch/README.md @@ -61,7 +61,7 @@ one should execute: ```bash git clone https://github.com/youngwanLEE/centermask2.git cd centermask2 -python3 train_net.py --config-file "configs/centermask/centermask_V_39_eSE_FPN_ms_3x.yaml" --num-gpus 8 +python3 train_net.py --config-file configs/centermask/centermask_R_50_FPN_ms_3x.yaml --num-gpus 8 ``` ## References diff --git a/cv/detection/centermask2/pytorch/ci/prepare.sh b/cv/detection/centermask2/pytorch/ci/prepare.sh new file mode 100644 index 00000000..419cbc32 --- /dev/null +++ b/cv/detection/centermask2/pytorch/ci/prepare.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +# Install detectron2 with 9604f5995cc628619f0e4fd913453b4d7d61db3f +git clone https://github.com/facebookresearch/detectron2.git +python3 -m pip install -e detectron2 + +# 6adabd66034347b7da07f2d474e4baa1c27b54ee is the commit hash of centermask2 +git clone https://github.com/youngwanLEE/centermask2.git +cd centermask2 +# fix RuntimeError: "max_elementwise_cuda" not implemented for 'Double' +sed -i 's@torch.max(mask_ratios, value_eps)@torch.max(mask_ratios.cpu(), value_eps.cpu()).cuda()@' centermask/modeling/centermask/mask_head.py +sed -i 's@torch.max(mask_union_area, value_1)@torch.max(mask_union_area.cpu(), value_1.cpu()).cuda()@' centermask/modeling/centermask/mask_head.py +sed -i 's@torch.max(mask_ovr_area, value_0)@torch.max(mask_ovr_area.cpu(), value_0.cpu()).cuda()@' centermask/modeling/centermask/mask_head.py + +mkdir -p datasets/ +ln -s /mnt/deepspark/data/datasets/coco2017 datasets/coco +mkdir -p /root/.torch/iopath_cache/detectron2/ImageNetPretrained/MSRA/ +cp /mnt/deepspark/data/checkpoints/R-50.pkl /root/.torch/iopath_cache/detectron2/ImageNetPretrained/MSRA/ +timeout 1800 python3 train_net.py --config-file configs/centermask/centermask_R_50_FPN_ms_3x.yaml --num-gpus 4 \ No newline at end of file diff --git a/cv/detection/centernet/paddlepaddle/README.md b/cv/detection/centernet/paddlepaddle/README.md index f1ad9a71..afec74dc 100644 --- a/cv/detection/centernet/paddlepaddle/README.md +++ b/cv/detection/centernet/paddlepaddle/README.md @@ -13,7 +13,13 @@ can be extended to 3D detection and pose estimation tasks. ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset @@ -52,3 +58,6 @@ python3 -m paddle.distributed.launch --gpus 0,1,2,3,4,5,6,7 tools/train.py -c co | GPU | learning rate | FPS | Train Epochs | mAP | |------------|---------------|-------|--------------|------| | BI-V100 x8 | 0.00005 | 10.85 | 3 | 38.5 | + +## Reference +[PaddleDetection](https://github.com/PaddlePaddle/PaddleDetection) \ No newline at end of file diff --git a/cv/detection/centernet/pytorch/README.md b/cv/detection/centernet/pytorch/README.md index 2b97bd94..dd0198cf 100644 --- a/cv/detection/centernet/pytorch/README.md +++ b/cv/detection/centernet/pytorch/README.md @@ -72,7 +72,7 @@ python3 setup.py build develop ## Model Training ```bash -cd ./cv/detection/centernet/pytorch/src +cd ./src touch lib/datasets/__init__.py export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 diff --git a/cv/detection/centernet/pytorch/ci/prepare.sh b/cv/detection/centernet/pytorch/ci/prepare.sh new file mode 100644 index 00000000..89047cc8 --- /dev/null +++ b/cv/detection/centernet/pytorch/ci/prepare.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +pip3 install -r requirements.txt +git clone https://github.com/xingyizhou/CenterNet.git +cd CenterNet +git checkout 4c50fd3a46bdf63dbf2082c5cbb3458d39579e6c +# Compile deformable convolutional(DCNv2) +cd ./src/lib/models/networks/ +rm -rf DCNv2 +git clone -b pytorch_1.11 https://github.com/lbin/DCNv2.git +cd ./DCNv2/ +python3 setup.py build develop +cd ../../../../../ +ln -s /mnt/deepspark/data/datasets/coco2017 ./data/coco +mkdir -p /root/.cache/torch/hub/checkpoints/ +cp /mnt/deepspark/data/checkpoints/resnet18-5c106cde.pth /root/.cache/torch/hub/checkpoints/ + +cd ./src +touch lib/datasets/__init__.py +timeout 1800 python3 main.py ctdet --arch res_18 --batch_size 32 --master_batch 15 --lr 1.25e-4 --gpus 0 \ No newline at end of file diff --git a/cv/detection/co-detr/pytorch/README.md b/cv/detection/co-detr/pytorch/README.md index c7978566..b73ec656 100644 --- a/cv/detection/co-detr/pytorch/README.md +++ b/cv/detection/co-detr/pytorch/README.md @@ -51,10 +51,6 @@ git clone https://github.com/open-mmlab/mmdetection.git -b v3.3.0 --depth=1 cd mmdetection pip install -v -e . -# Install requirements -pip3 install -r requirements.txt -pip3 install urllib3==1.26.15 - # Download repo git clone https://github.com/Sense-X/Co-DETR.git cd /path/to/Co-DETR @@ -72,12 +68,9 @@ ln -s /path/to/coco ./data export CUDA_VISIBLE_DEVICES=0 python3 tools/train.py projects/configs/co_deformable_detr/co_deformable_detr_r50_1x_coco.py --work-dir path_to_exp --no-validate --auto-resume -# Eight GPUs -bash tools/dist_train.sh projects/configs/co_deformable_detr/co_deformable_detr_r50_1x_coco.py 8 path_to_exp --no-validate --auto-resume - -# Evaluation -export CUDA_VISIBLE_DEVICES=0 -PYTHONPATH=".:$PYTHONPATH" python3 tools/test.py projects/configs/co_deformable_detr/co_deformable_detr_r50_1x_coco.py path_to_exp/latest.pth --eval bbox +sed -i 's/python /python3 /g' tools/dist_train.sh +# Multiple GPUs on one machine +bash tools/dist_train.sh projects/CO-DETR/configs/codino/co_dino_5scale_r50_lsj_8xb2_1x_coco.py 8 ``` ## Model Results @@ -86,7 +79,5 @@ PYTHONPATH=".:$PYTHONPATH" python3 tools/test.py projects/configs/co_deformable_ |---------|------------|------|--------------|--------| | Co-DETR | BI-V100 x8 | 9.02 | 12 | 0.428 | -## References - -- [Paper](https://arxiv.org/pdf/2211.12860.pdf) -- [Co-DETR](https://github.com/Sense-X/Co-DETR) +## Reference +[mmdetection](https://github.com/open-mmlab/mmdetection) diff --git a/cv/detection/co-detr/pytorch/requirements.txt b/cv/detection/co-detr/pytorch/requirements.txt deleted file mode 100644 index 70e3bb13..00000000 --- a/cv/detection/co-detr/pytorch/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -addict -pycocotools -terminaltables -yapf==0.40.1 -fairscale -timm diff --git a/cv/detection/cornernet_mmdet/pytorch/README.md b/cv/detection/cornernet_mmdet/pytorch/README.md index 169c9c8c..ffaf9595 100644 --- a/cv/detection/cornernet_mmdet/pytorch/README.md +++ b/cv/detection/cornernet_mmdet/pytorch/README.md @@ -78,6 +78,5 @@ bash tools/dist_train.sh configs/cornernet/cornernet_hourglass104_8xb6-210e-mste |-----------|------------|----------| | CornerNet | BI-V100 x8 | MAP=41.2 | -## References - -- [Paper](https://arxiv.org/abs/1808.01244) +## Reference +[mmdetection](https://github.com/open-mmlab/mmdetection) diff --git a/cv/detection/dcnv2_mmdet/pytorch/README.md b/cv/detection/dcnv2_mmdet/pytorch/README.md index 4f7f8595..8977ca8f 100644 --- a/cv/detection/dcnv2_mmdet/pytorch/README.md +++ b/cv/detection/dcnv2_mmdet/pytorch/README.md @@ -78,6 +78,5 @@ bash tools/dist_train.sh configs/dcnv2/faster-rcnn_r50-mdconv-c3-c5_fpn_1x_coco. |-------|------------|------| | DCNv2 | BI-V100 x8 | 41.2 | -## References - -- [Paper](https://arxiv.org/abs/1811.11168) +## Reference +[mmdetection](https://github.com/open-mmlab/mmdetection) diff --git a/cv/detection/detr/paddlepaddle/README.md b/cv/detection/detr/paddlepaddle/README.md index 326b1b5a..701b4f8b 100644 --- a/cv/detection/detr/paddlepaddle/README.md +++ b/cv/detection/detr/paddlepaddle/README.md @@ -13,7 +13,13 @@ while achieving competitive performance on benchmarks like COCO, offering a new ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset @@ -52,3 +58,7 @@ python3 -m paddle.distributed.launch --gpus 0,1,2,3,4,5,6,7 tools/train.py -c co | Model | GPU | learning rate | FPS | Train Epochs | Box AP | |-------|------------|---------------|-------|--------------|--------| | DETR | BI-V100 x8 | 0.00001 | 14.64 | 1 | 42.0 | + +## Reference + +- [PaddleDetection](https://github.com/PaddlePaddle/PaddleDetection) \ No newline at end of file diff --git a/cv/detection/fasterrcnn/pytorch/ci/prepare.sh b/cv/detection/fasterrcnn/pytorch/ci/prepare.sh new file mode 100644 index 00000000..1a769f5c --- /dev/null +++ b/cv/detection/fasterrcnn/pytorch/ci/prepare.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +mkdir -p data/ +ln -s /mnt/deepspark/data/datasets/coco2017 data/coco +mkdir -p /root/.cache/torch/hub/checkpoints/ +cp /mnt/deepspark/data/checkpoints/resnet50-0676ba61.pth /root/.cache/torch/hub/checkpoints/ + +cd start_scripts +bash init_torch.sh + +timeout 1800 bash train_fasterrcnn_resnet50_amp_torch.sh --dataset coco --data-path ../data/coco + +# bash train_fasterrcnn_resnet50_amp_dist_torch.sh --dataset coco --data-path ../data/coco diff --git a/cv/detection/fcos/paddlepaddle/README.md b/cv/detection/fcos/paddlepaddle/README.md index 877b88a6..bede6c79 100644 --- a/cv/detection/fcos/paddlepaddle/README.md +++ b/cv/detection/fcos/paddlepaddle/README.md @@ -13,7 +13,13 @@ it efficient and effective for various detection tasks. ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset @@ -47,3 +53,7 @@ python3 tools/train.py -c configs/fcos/fcos_r50_fpn_1x_coco.yml --eval | Model | GPU | FPS | Train Epochs | Box AP | |-------|------------|------|--------------|--------| | FCOS | BI-V100 x8 | 8.24 | 12 | 39.7 | + +## Reference + +- [PaddleDetection](https://github.com/PaddlePaddle/PaddleDetection) \ No newline at end of file diff --git a/cv/detection/fcos/pytorch/ci/prepare.sh b/cv/detection/fcos/pytorch/ci/prepare.sh new file mode 100644 index 00000000..da77dbf3 --- /dev/null +++ b/cv/detection/fcos/pytorch/ci/prepare.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +pip3 install -r requirements.txt +pip uninstall numpy +pip install numpy==1.23.5 +python3 setup.py develop + +mkdir -p data +ln -s /mnt/deepspark/data/datasets/coco2017 data/coco +mkdir -p /root/.torch/models/ +cp /mnt/deepspark/data/checkpoints/R-50.pkl /root/.torch/models/ + +timeout 1800 python3 -m torch.distributed.launch \ + --nproc_per_node=4 \ + tools/train_net.py \ + --config-file configs/fcos/fcos_imprv_R_50_FPN_1x.yaml \ + DATALOADER.NUM_WORKERS 2 \ + OUTPUT_DIR training_dir/fcos_imprv_R_50_FPN_1x \ No newline at end of file diff --git a/cv/detection/mamba_yolo/pytorch/README.md b/cv/detection/mamba_yolo/pytorch/README.md index 29d77994..be6a2484 100644 --- a/cv/detection/mamba_yolo/pytorch/README.md +++ b/cv/detection/mamba_yolo/pytorch/README.md @@ -49,9 +49,8 @@ vim ultralytics/cfg/datasets/coco.yaml ```sh pip3 install seaborn thop timm einops - git clone --depth 1 https://gitee.com/deep-spark/deepsparkhub-GPL.git -cd cv/detection/mamba-yolo/pytorch +cd deepsparkhub-GPL/cv/detection/mamba-yolo/pytorch cd selective_scan && pip install . && cd .. pip install -v -e . diff --git a/cv/detection/mamba_yolo/pytorch/ci/prepare.sh b/cv/detection/mamba_yolo/pytorch/ci/prepare.sh new file mode 100644 index 00000000..81c88949 --- /dev/null +++ b/cv/detection/mamba_yolo/pytorch/ci/prepare.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +pip3 install seaborn thop timm einops + +mkdir -p /root/.config/Ultralytics/ +cp /mnt/deepspark/data/3rd_party/Arial.ttf /root/.config/Ultralytics/ + +git clone --depth 1 https://gitee.com/deep-spark/deepsparkhub-GPL.git +cd deepsparkhub-GPL/cv/detection/mamba-yolo/pytorch + +cd selective_scan && pip install . && cd .. +pip install -v -e . + +mkdir -p /root/data +ln -s /mnt/deepspark/data/datasets/coco /root/data/coco +sed -i 's/\/mnt\/datasets\/MSCOCO2017/\/root\/data\/coco/g' ultralytics/cfg/datasets/coco.yaml +ln -s /mnt/deepspark/data/checkpoints/yolov8n.pt ./ + +timeout 1800 python3 mbyolo_train.py --task train --data ultralytics/cfg/datasets/coco.yaml \ + --config ultralytics/cfg/models/mamba-yolo/Mamba-YOLO-T.yaml \ +--amp --project ./output_dir/mscoco --name mambayolo_n --device 0,1,2,3 \ No newline at end of file diff --git a/cv/detection/maskrcnn/paddlepaddle/README.md b/cv/detection/maskrcnn/paddlepaddle/README.md index 7efd0413..f98a7cd6 100644 --- a/cv/detection/maskrcnn/paddlepaddle/README.md +++ b/cv/detection/maskrcnn/paddlepaddle/README.md @@ -13,11 +13,20 @@ detection, instance segmentation, and human pose estimation. ### Prepare Resources ```bash +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone --recursive https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 git clone https://github.com/PaddlePaddle/PaddleDetection.git cd PaddleDetection/ # Get COCO Dataset python3 dataset/coco/download_coco.py +or +ln -s /path/to/coco2017 dataset/coco ``` ### Install Dependencies @@ -42,3 +51,7 @@ python3 -u -m paddle.distributed.launch --gpus 0,1,2,3,4,5,6,7 tools/train.py -c | Model | GPU | FP32 | |------------|------------|-------------------------------| | Mask R-CNN | BI-V100 x8 | bbox=38.8,FPS=7.5,BatchSize=1 | + +## Reference + +- [PaddleDetection](https://github.com/PaddlePaddle/PaddleDetection) \ No newline at end of file diff --git a/cv/detection/maskrcnn/pytorch/ci/prepare.sh b/cv/detection/maskrcnn/pytorch/ci/prepare.sh new file mode 100644 index 00000000..b2887561 --- /dev/null +++ b/cv/detection/maskrcnn/pytorch/ci/prepare.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +pip3 install -r requirements.txt + +mkdir -p ./datasets/ +ln -s /mnt/deepspark/data/datasets/coco2017 ./datasets/coco +mkdir -p /root/.cache/torch/hub/checkpoints/ +cp /mnt/deepspark/data/checkpoints/resnet50-0676ba61.pth /root/.cache/torch/hub/checkpoints/resnet50-0676ba61.pth + +### Single Card +timeout 1800 python3 train.py --data-path ./datasets/coco --dataset coco --model maskrcnn_resnet50_fpn --lr 0.001 --batch-size 4 + +### AMP +# python3 train.py --data-path ./datasets/coco --dataset coco --model maskrcnn_resnet50_fpn --lr 0.001 --batch-size 1 --amp + +### DDP + +# python3 -m torch.distributed.launch --nproc_per_node=8 --use_env train.py\ + # --data-path ./datasets/coco --dataset coco --model maskrcnn_resnet50_fpn --wd 0.000001 --lr 0.001 --batch-size 4 \ No newline at end of file diff --git a/cv/detection/oc_sort/paddlepaddle/README.md b/cv/detection/oc_sort/paddlepaddle/README.md index 3dcc3136..2d8a0559 100644 --- a/cv/detection/oc_sort/paddlepaddle/README.md +++ b/cv/detection/oc_sort/paddlepaddle/README.md @@ -152,9 +152,13 @@ ln -s ../ByteTrack/datasets/mot/annotations ./ ### Install Dependencies ```bash -yum install mesa-libGL -y +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx -git clone https://github.com/PaddlePaddle/PaddleDetection.git +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ pip3 install -r requirements.txt diff --git a/cv/detection/picodet/paddlepaddle/README.md b/cv/detection/picodet/paddlepaddle/README.md index 5bcdedcb..cee3cff1 100644 --- a/cv/detection/picodet/paddlepaddle/README.md +++ b/cv/detection/picodet/paddlepaddle/README.md @@ -13,7 +13,13 @@ on resource-constrained devices. ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset diff --git a/cv/detection/pp-yoloe/paddlepaddle/README.md b/cv/detection/pp-yoloe/paddlepaddle/README.md index 817b1667..eed132ea 100644 --- a/cv/detection/pp-yoloe/paddlepaddle/README.md +++ b/cv/detection/pp-yoloe/paddlepaddle/README.md @@ -14,11 +14,19 @@ for object detection tasks. ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset python3 dataset/coco/download_coco.py +or +ln -s /path/to/coco2017 dataset/coco ``` ### Install Dependencies diff --git a/cv/detection/pp_yoloe+/paddlepaddle/README.md b/cv/detection/pp_yoloe+/paddlepaddle/README.md index 851a0524..e724119b 100644 --- a/cv/detection/pp_yoloe+/paddlepaddle/README.md +++ b/cv/detection/pp_yoloe+/paddlepaddle/README.md @@ -13,10 +13,18 @@ real-time applications, offering superior detection performance across various s ### Prepare Resources ```bash +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + git clone -b release/2.7 https://github.com/PaddlePaddle/PaddleYOLO.git cd PaddleYOLO/ python3 dataset/coco/download_coco.py +or +ln -s /path/to/coco2017 dataset/coco ``` ### Install Dependencies diff --git a/cv/detection/pvanet/pytorch/ci/prepare.sh b/cv/detection/pvanet/pytorch/ci/prepare.sh new file mode 100644 index 00000000..42ca27f1 --- /dev/null +++ b/cv/detection/pvanet/pytorch/ci/prepare.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +pip3 install 'scipy' 'matplotlib' 'pycocotools' 'opencv-python' 'easydict' 'tqdm' + +### Multiple GPUs on one machine +# bash train_pvanet_dist.sh --data-path /mnt/deepspark/data/datasets/coco2017/ --dataset coco + +### On single GPU +timeout 1800 python3 train.py --data-path /mnt/deepspark/data/datasets/coco2017/ --dataset coco \ No newline at end of file diff --git a/cv/detection/pvanet/pytorch/coco_eval.py b/cv/detection/pvanet/pytorch/coco_eval.py index a6e5547c..8004f54b 100755 --- a/cv/detection/pvanet/pytorch/coco_eval.py +++ b/cv/detection/pvanet/pytorch/coco_eval.py @@ -5,7 +5,7 @@ import numpy as np import copy import time import torch -import torch._six +# import torch._six # pytorch 2.0后无此api from pycocotools.cocoeval import COCOeval from pycocotools.coco import COCO diff --git a/cv/detection/reppoints_mmdet/pytorch/README.md b/cv/detection/reppoints_mmdet/pytorch/README.md index b8849f69..82370f5b 100644 --- a/cv/detection/reppoints_mmdet/pytorch/README.md +++ b/cv/detection/reppoints_mmdet/pytorch/README.md @@ -77,6 +77,5 @@ bash tools/dist_train.sh configs/reppoints/reppoints-moment_r101-dconv-c3-c5_fpn |-----------|------------|----------| | RepPoints | BI-V100 x8 | MAP=43.2 | -## References - -- [Paper](https://arxiv.org/abs/1904.11490) +## Reference +[mmdetection](https://github.com/open-mmlab/mmdetection) diff --git a/cv/detection/retinanet/paddlepaddle/README.md b/cv/detection/retinanet/paddlepaddle/README.md index 11e4287a..9a007f67 100644 --- a/cv/detection/retinanet/paddlepaddle/README.md +++ b/cv/detection/retinanet/paddlepaddle/README.md @@ -13,7 +13,13 @@ single-stage approaches, offering an excellent balance between performance and e ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset diff --git a/cv/detection/retinanet/pytorch/ci/prepare.sh b/cv/detection/retinanet/pytorch/ci/prepare.sh new file mode 100644 index 00000000..bd4de78a --- /dev/null +++ b/cv/detection/retinanet/pytorch/ci/prepare.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +pip3 install 'scipy' 'matplotlib' 'pycocotools' 'opencv-python' 'easydict' 'tqdm' + +mkdir -p /root/.cache/torch/hub/checkpoints/ +cp /mnt/deepspark/data/checkpoints/resnet50-19c8e357.pth /root/.cache/torch/hub/checkpoints/resnet50-19c8e357.pth + +timeout 1800 bash train_retinanet_r50_dist.sh --data-path /mnt/deepspark/data/datasets/coco2017/ --dataset coco \ No newline at end of file diff --git a/cv/detection/rt-detr/pytorch/ci/prepare.sh b/cv/detection/rt-detr/pytorch/ci/prepare.sh new file mode 100644 index 00000000..24a33219 --- /dev/null +++ b/cv/detection/rt-detr/pytorch/ci/prepare.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +pip3 install -r requirements.txt + +mkdir -p dataset +ln -s /mnt/deepspark/data/datasets/coco2017 dataset/coco + +# training on single-gpu +export CUDA_VISIBLE_DEVICES=0 +timeout 1800 python3 tools/train.py -c configs/rtdetr/rtdetr_r50vd_6x_coco.yml + +# train on multi-gpu +# export CUDA_VISIBLE_DEVICES=0,1,2,3 +# torchrun --nproc_per_node=4 tools/train.py -c configs/rtdetr/rtdetr_r50vd_6x_coco.yml + +# val on multi-gpu +# export CUDA_VISIBLE_DEVICES=0,1,2,3 +# torchrun --nproc_per_node=4 tools/train.py -c configs/rtdetr/rtdetr_r50vd_6x_coco.yml -r path/to/checkpoint --test-only \ No newline at end of file diff --git a/cv/detection/solov2/paddlepaddle/README.md b/cv/detection/solov2/paddlepaddle/README.md index 2cda7326..8213cdc6 100644 --- a/cv/detection/solov2/paddlepaddle/README.md +++ b/cv/detection/solov2/paddlepaddle/README.md @@ -13,7 +13,13 @@ tasks while maintaining real-time capabilities, making it suitable for various c ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset @@ -47,3 +53,7 @@ python3 tools/train.py -c configs/solov2/solov2_r50_fpn_1x_coco.yml --eval | Model | GPU | FPS | Train Epochs | mAP | |--------|------------|------|--------------|------| | SOLOv2 | BI-V100 x8 | 6.39 | 12 | 35.4 | + +## Reference + +- [PaddleDetection](https://github.com/PaddlePaddle/PaddleDetection) \ No newline at end of file diff --git a/cv/detection/ssd/paddlepaddle/README.md b/cv/detection/ssd/paddlepaddle/README.md index ce64580a..306f9c2c 100644 --- a/cv/detection/ssd/paddlepaddle/README.md +++ b/cv/detection/ssd/paddlepaddle/README.md @@ -12,11 +12,19 @@ objects at different resolutions, offering a good balance between speed and accu ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset python3 dataset/coco/download_coco.py +or +ln -s /path/to/coco2017 dataset/coco ``` ### Install Dependencies diff --git a/cv/detection/ssd/pytorch/base/prepare.py b/cv/detection/ssd/pytorch/base/prepare.py index b85fb752..4b181436 100644 --- a/cv/detection/ssd/pytorch/base/prepare.py +++ b/cv/detection/ssd/pytorch/base/prepare.py @@ -6,7 +6,7 @@ import os.path as ospath from argparse import ArgumentParser, REMAINDER from functools import partial, wraps from typing import NamedTuple -import platform +# import platform # python3.5已弃用 # ========================================================= @@ -268,12 +268,27 @@ def install_extensions(): return exec_shell_cmds(cmds) +def get_linux_distro(): + try: + with open('/etc/os-release') as f: + for line in f: + if line.startswith('NAME='): + # 提取 NAME 字段的值（例如 "Ubuntu" 或 "CentOS"） + name = line.split('=')[1].strip().strip('"') + if 'Ubuntu' in name: + return 'Ubuntu' + elif 'CentOS' in name: + return 'CentOS' + else: + return name # 返回其他发行版名称 + except FileNotFoundError: + return 'Unknown Linux distribution' def pipelines(): - plat = [x.lower() for x in platform.platform().split("-")] - if "centos" in plat: + plat = get_linux_distro().lower() + if "centos" == plat: res = [install_yum_packages] - elif "debian" in plat or "ubuntu" in plat: + elif "ubuntu" == plat: res = [install_apt_packages] else: raise Exception("Invalid Platform, only support Centos and Debian!") diff --git a/cv/detection/ssd/pytorch/ci/prepare.sh b/cv/detection/ssd/pytorch/ci/prepare.sh new file mode 100644 index 00000000..7a9d113e --- /dev/null +++ b/cv/detection/ssd/pytorch/ci/prepare.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +mkdir -p /home/data/perf/ssd +ln -s /mnt/deepspark/data/datasets/coco /home/data/perf/ssd/ +cp /mnt/deepspark/data/checkpoints/resnet34-333f7ec4.pth /home/data/perf/ssd/ + +cd base +source ../iluvatar/config/environment_variables.sh +python3 prepare.py --name iluvatar --data_dir /home/data/perf/ssd +timeout 1800 bash run_training.sh --name iluvatar --config V100x1x8 --data_dir /home/data/perf/ssd --backbone_path /home/data/perf/ssd/resnet34-333f7ec4.pth \ No newline at end of file diff --git a/cv/detection/ssd/tensorflow/README.md b/cv/detection/ssd/tensorflow/README.md index fe9ff61d..921a0966 100644 --- a/cv/detection/ssd/tensorflow/README.md +++ b/cv/detection/ssd/tensorflow/README.md @@ -32,6 +32,13 @@ dataset/VOCROOT/ VOCROOT is your path of the Pascal VOC Dataset. ```bash +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +cd dataset mkdir tfrecords pip3 install tf_slim python3 convert_voc_sample_tfrecords.py --dataset_directory=./ --output_directory=tfrecords --train_splits VOC2012_sample --validation_splits VOC2012_sample diff --git a/cv/detection/ssd/tensorflow/ci/prepare.sh b/cv/detection/ssd/tensorflow/ci/prepare.sh new file mode 100644 index 00000000..363671f6 --- /dev/null +++ b/cv/detection/ssd/tensorflow/ci/prepare.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +pip3 install tf_slim +cd dataset +mkdir tfrecords +ln -s /mnt/deepspark/data/datasets/VOC2012_sample ./ +python3 convert_voc_sample_tfrecords.py --dataset_directory=./ --output_directory=tfrecords --train_splits VOC2012_sample --validation_splits VOC2012_sample +cd .. +ln -s /mnt/deepspark/data/checkpoints/ssd-vgg16 ./model +python3 train_ssd.py --batch_size 16 \ No newline at end of file diff --git a/cv/detection/yolov3/paddlepaddle/README.md b/cv/detection/yolov3/paddlepaddle/README.md index 5c70713a..b7f6cead 100644 --- a/cv/detection/yolov3/paddlepaddle/README.md +++ b/cv/detection/yolov3/paddlepaddle/README.md @@ -13,11 +13,19 @@ practical detection tasks. ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone --recursive https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset python3 dataset/coco/download_coco.py +or +ln -s /path/to/coco2017 dataset/coco ``` ### Install Dependencies @@ -42,3 +50,7 @@ python3 -u -m paddle.distributed.launch --gpus 0,1,2,3,4,5,6,7 \ --use_vdl=true \ --eval ``` + +## Reference + +- [PaddleDetection](https://github.com/PaddlePaddle/PaddleDetection) \ No newline at end of file diff --git a/cv/detection/yolov3/tensorflow/README.md b/cv/detection/yolov3/tensorflow/README.md index 92d9619c..ccfbf083 100644 --- a/cv/detection/yolov3/tensorflow/README.md +++ b/cv/detection/yolov3/tensorflow/README.md @@ -40,6 +40,12 @@ Exporting loaded COCO weights as TF checkpoint(yolov3_coco.ckpt)[BaiduCloud](htt ### Install Dependencies ```bash +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + bash init_tf.sh ``` diff --git a/cv/detection/yolov3/tensorflow/ci/prepare.sh b/cv/detection/yolov3/tensorflow/ci/prepare.sh new file mode 100644 index 00000000..f5329ce7 --- /dev/null +++ b/cv/detection/yolov3/tensorflow/ci/prepare.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +bash init_tf.sh +ln -s /mnt/deepspark/data/datasets/VOC ./ +bash ./run_training.sh \ No newline at end of file diff --git a/cv/detection/yolov3/tensorflow/train_fast.py b/cv/detection/yolov3/tensorflow/train_fast.py index 5992fd4a..9217bd05 100644 --- a/cv/detection/yolov3/tensorflow/train_fast.py +++ b/cv/detection/yolov3/tensorflow/train_fast.py @@ -227,6 +227,8 @@ class YoloTrain(object): self.summary_writer.add_summary(summary, global_step_val) pbar.set_description("train loss: %.2f" % train_step_loss) + # fix epoch not assign + epoch = 1+self.first_stage_epochs+self.second_stage_epochs train_epoch_loss = np.mean(train_epoch_loss) log_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) print("=> Epoch: %2d Time: %s Train loss: %.2f..." diff --git a/cv/detection/yolov5/paddlepaddle/README.md b/cv/detection/yolov5/paddlepaddle/README.md index 4e2b338f..dc7b4891 100644 --- a/cv/detection/yolov5/paddlepaddle/README.md +++ b/cv/detection/yolov5/paddlepaddle/README.md @@ -12,11 +12,18 @@ and efficient inference, making it popular for real-time detection tasks across ### Prepare Resources ```bash -cd deepsparkhub/cv/detection/yolov5/paddlepaddle/ -git clone -b release/2.5 https://github.com/PaddlePaddle/PaddleYOLO.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone -b release/2.7 --depth=1 https://github.com/PaddlePaddle/PaddleYOLO.git cd PaddleYOLO/ python3 dataset/coco/download_coco.py +or +ln -s /path/to/coco2017 dataset/coco ``` ### Install Dependencies diff --git a/cv/detection/yolov5/pytorch/ci/prepare.sh b/cv/detection/yolov5/pytorch/ci/prepare.sh new file mode 100644 index 00000000..bc538008 --- /dev/null +++ b/cv/detection/yolov5/pytorch/ci/prepare.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +## clone yolov5 and install +git clone https://gitee.com/deep-spark/deepsparkhub-GPL.git +cd deepsparkhub-GPL/cv/detection/yolov5/pytorch/ +pip3 install -r requirements.txt +pip3 install matplotlib==3.8.2 +pip3 install numpy==1.26.4 +pip3 install pandas==2.0.0 +wandb disabled +pip3 install pycocotools + +mkdir -p datasets +ln -s /mnt/deepspark/data/datasets/coco ./datasets/coco + +### On single GPU +timeout 1800 python3 train.py --data ./data/coco.yaml --batch-size 32 --cfg ./models/yolov5s.yaml --weights '' + +### On single GPU (AMP) +# python3 train.py --data ./data/coco.yaml --batch-size 32 --cfg ./models/yolov5s.yaml --weights '' --amp + +### Multiple GPUs on one machine +# python3 -m torch.distributed.launch --nproc_per_node 8 \ +# train.py \ +# --data ./data/coco.yaml \ +# --batch-size 64 \ +# --cfg ./models/yolov5s.yaml --weights '' \ +# --device 0,1,2,3,4,5,6,7 + +# YOLOv5m +# bash run.sh + +# ### Multiple GPUs on one machine (AMP) +# python3 -m torch.distributed.launch --nproc_per_node 8 \ +# train.py \ +# --data ./data/coco.yaml \ +# --batch-size 256 \ +# --cfg ./models/yolov5s.yaml --weights '' \ +# --device 0,1,2,3,4,5,6,7 --amp + + +## Test the detector +# python3 detect.py --source ./data/images/bus.jpg --weights yolov5s.pt --img 640 +# python3 detect.py --source ./data/images/zidane.jpg --weights yolov5s.pt --img 640 \ No newline at end of file diff --git a/tests/run_test.py b/tests/run_test.py new file mode 100644 index 00000000..cd0146d6 --- /dev/null +++ b/tests/run_test.py @@ -0,0 +1,300 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml +import subprocess +import json +import re +import time +import logging +import os +import sys +import argparse + +def is_debug(): + is_debug_flag = os.environ.get("IS_DEBUG") + if is_debug_flag and is_debug_flag.lower()=="true": + return True + else: + return False + +def main(): + parser = argparse.ArgumentParser(description="") + parser.add_argument("--model", type=str, help="model name, e.g: alexnet") + args = parser.parse_args() + + if args.model: + test_model = args.model + else: + test_model = os.environ.get("TEST_CASE") + test_framework = os.environ.get("FRAMEWORK") + test_category = os.environ.get("category") + logging.info(f"Test case to run: {test_model}") + if not test_model: + logging.error("test model case is empty") + sys.exit(-1) + + model = get_model_config(test_model, test_framework, test_category) + if not model: + logging.error("mode config is empty") + sys.exit(-1) + + result = {} + if model["category"] == "cv/classification" and model["framework"] == "pytorch": + logging.info(f"Start running {model['model_name']} test case:\n{json.dumps(model, indent=4)}") + result = run_clf_testcase(model) + check_model_result(result) + logging.debug(f"The result of {model['model_name']} is\n{json.dumps(result, indent=4)}") + logging.info(f"End running {model['model_name']} test case.") + + if model["category"] == "cv/detection" and (model["framework"] == "pytorch" or model["framework"] == "paddlepaddle"): + logging.info(f"Start running {model['model_name']} test case:\n{json.dumps(model, indent=4)}") + result = run_detec_testcase(model) + check_model_result(result) + logging.debug(f"The result of {model['model_name']} is\n{json.dumps(result, indent=4)}") + logging.info(f"End running {model['model_name']} test case.") + + logging.info(f"Full text result: {result}") + +def get_model_config(model_name, framework, category): + print(f"model_name: {model_name}, framework: {framework}, category: {category}") + with open('all_deepsparkhub_model_info.json', mode='r', encoding='utf-8') as config_file: + config_data = json.load(config_file) + + for model in config_data["models"]: + # TODO: 模型名称+模型框架+模型类别才能唯一确定一个模型 + if model["model_name"] == model_name.lower() and model["framework"] == framework.lower() and model["category"] == category.lower(): + return model + return + +def check_model_result(result): + status = "PASS" + for prec in ["fp16", "int8"]: + if prec in result["result"]: + if result["result"][prec]["status"] == "FAIL": + status = "FAIL" + break + result["status"] = status + +def run_detec_testcase(model): + model_name = model["model_name"] + result = { + "name": model_name, + "framework": model["framework"], + "toolbox": model["toolbox"], + "category": "cv/detection", + "result": {}, + } + is_mmdetection = True if model["toolbox"] == "mmdetection" else False + is_yolov = True if model["toolbox"] == "yolov" else False + is_paddledetection = True if model["toolbox"] == "PaddleDetection" else False + deepsparkhub_path = model["deepsparkhub_path"].replace("deepsparkhub/", "") + if is_mmdetection: + # 选择使用atss作为个例 + prepare_script = f""" + apt install -y libgl1-mesa-glx + cd ../cv/detection/atss_mmdet/pytorch/ + cp -r /mnt/deepspark/data/3rd_party/mmdetection-v3.3.0 ./mmdetection > /dev/null 2>&1 + mkdir -p /root/.cache/torch/hub/checkpoints/ + cp /mnt/deepspark/data/checkpoints/resnet50-0676ba61.pth /root/.cache/torch/hub/checkpoints/resnet50-0676ba61.pth + mkdir -p mmdetection/data + ln -s /mnt/deepspark/data/datasets/coco2017 mmdetection/data/coco + cd mmdetection + pip install -v -e . + sed -i 's/python /python3 /g' tools/dist_train.sh + timeout 1800 bash tools/dist_train.sh configs/atss/atss_r50_fpn_1x_coco.py 4 + """ + elif is_paddledetection: + # 选择使用CenterNet作为个例 + prepare_script = f""" + apt install -y libgl1-mesa-glx + cd ../cv/detection/centernet/paddlepaddle/ + cp -r /mnt/deepspark/data/3rd_party/PaddleDetection-release-2.6 ./PaddleDetection + cd PaddleDetection + pip install -r requirements.txt + python3 setup.py install + mv dataset/coco dataset/coco-bak + ln -s /mnt/deepspark/data/datasets/coco2017 dataset/coco + timeout 1800 python3 tools/train.py -c configs/centernet/centernet_r50_140e_coco.yml --eval + """ + elif is_yolov: + # 选择使用yolov5作为个例 + prepare_script = f""" + cd ../cv/detection/yolov5/pytorch/ + bash ci/prepare.sh + """ + else: + prepare_script = f""" + cd ../{deepsparkhub_path} + bash ci/prepare.sh + """ + + # add pip list info when in debug mode + if is_debug(): + pip_list_script = "pip list | grep -E 'numpy|transformer|igie|mmcv|onnx'\n" + prepare_script = pip_list_script + prepare_script + pip_list_script + + logging.info(f"Start running {model_name} test case") + r, t = run_script(prepare_script) + sout = r.stdout + prec = "fp16" + pattern = r"Average Precision \(AP\) @\[ (IoU=0.50[:\d.]*)\s*\| area= all \| maxDets=\s?\d+\s?\] =\s*([\d.]+)" + matches = re.findall(pattern, sout) + epoch_pattern = [r"train: \[", r"Epoch: \[", r".*Epoch\s+gpu_mem", r"total_loss: "] + combined_pattern = re.compile("|".join(epoch_pattern)) + epoch_matches = bool(combined_pattern.search(sout)) + + if matches: + # Get last match and convert to floats + last_map = map(float, matches[-1]) + print(f"MAP: {last_map}") + result["result"].setdefault(prec, {"status": "PASS"}) + result["result"][prec]["MAP"] = last_map + elif epoch_matches: + result["result"].setdefault(prec, {"status": "PASS"}) + result["result"][prec]["MAP"] = "train timeout" + else: + result["result"].setdefault(prec, {"status": "FAIL"}) + print("No match found.") + + result["result"][prec]["Cost time (s)"] = t + logging.debug(f"matchs:\n{matches}\nepoch_matches:\n{epoch_matches}") + return result + +def run_clf_testcase(model): + model_name = model["model_name"] + result = { + "name": model_name, + "framework": model["framework"], + "toolbox": model["toolbox"], + "category": "cv/classification", + "result": {}, + } + is_torchvision = True if model["toolbox"].lower() == "torchvision" else False + is_mmpretrain = True if model["toolbox"].lower() == "mmpretrain" else False + is_paddleclas = True if model["toolbox"].lower() == "paddleclas" else False + is_tf_benchmarks = True if model["toolbox"].lower() == "tensorflow/benchmarks" else False + deepsparkhub_path = model["deepsparkhub_path"].replace("deepsparkhub/", "") + dataset_path = "/mnt/deepspark/data/datasets/imagenet" + + # # add pip list info when in debug mode + # if is_debug(): + # pip_list_script = "pip list | grep -E 'numpy|transformer|igie|mmcv|onnx'\n" + # prepare_script = pip_list_script + prepare_script + pip_list_script + + logging.info(f"Start running {model_name} test case") + if is_torchvision: + # 选择使用googlenet作为个例 + prepare_script = f""" + cd ../{deepsparkhub_path} + timeout 1800 python3 -m torch.distributed.launch --nproc_per_node=4 --use_env train.py --data-path {dataset_path} --model googlenet --batch-size 512 + """ + elif is_mmpretrain: + # 选择使用mocov2作为个例 + prepare_script = f""" + apt install -y libgl1-mesa-glx + cd ../cv/classification/mocov2/pytorch/ + cp -r /mnt/deepspark/data/3rd_party/mmpretrain-v1.2.0 ./mmpretrain > /dev/null 2>&1 + cd mmpretrain + python3 setup.py install + mkdir -p data + ln -s /mnt/deepspark/data/datasets/imagenet data/ + ln -s /mnt/deepspark/data/checkpoints/mocov2_resnet50_8xb32-coslr-200e_in1k_20220825-b6d23c86.pth ./ + sed -i 's@model = dict(backbone=dict(frozen_stages=4))@model = dict(backbone=dict(frozen_stages=4,init_cfg=dict(type='\''Pretrained'\'', checkpoint='\''./mocov2_resnet50_8xb32-coslr-200e_in1k_20220825-b6d23c86.pth'\'', prefix='\''backbone.'\'')))@' configs/mocov2/benchmarks/resnet50_8xb32-linear-steplr-100e_in1k.py + timeout 1800 python3 tools/train.py configs/mocov2/mocov2_resnet50_8xb32-coslr-200e_in1k.py + """ + elif is_paddleclas: + # 选择使用googlenet作为个例 + prepare_script = f""" + apt install -y libgl1-mesa-glx + cd ../cv/classification/googlenet/paddlepaddle/ + cp -r /mnt/deepspark/data/3rd_party/PaddleClas-release-2.6 ./PaddleClas > /dev/null 2>&1 + cd PaddleClas + pip3 install -r requirements.txt + python3 setup.py install + mkdir -p dataset + ln -s /mnt/deepspark/data/datasets/imagenet dataset/ILSVRC2012 + timeout 1800 python3 -u -m paddle.distributed.launch --gpus=0,1,2,3 tools/train.py -c ppcls/configs/ImageNet/Inception/GoogLeNet.yaml -o Arch.pretrained=False -o Global.device=gpu + """ + elif is_tf_benchmarks: + # 选择使用alexnet作为个例 + prepare_script = f""" + apt install -y libgl1-mesa-glx + cd ../cv/classification/alexnet/tensorflow/ + ln -s /mnt/deepspark/data/datasets/imagenet_tfrecord/ILSVRC2012 dataset/imagenet_tfrecord + timeout 1800 bash run_train_alexnet_imagenet.sh + """ + else: + prepare_script = f""" + cd ../{deepsparkhub_path} + ln -s /mnt/deepspark/data/datasets/imagenet ./ + bash ci/prepare.sh + """ + r, t = run_script(prepare_script) + sout = r.stdout + prec = "fp16" + pattern = re.compile(r'\* Acc@1 (\d+\.\d+) Acc@5 (\d+\.\d+)') + matches = pattern.findall(sout) + epoch_pattern = [r"Epoch: \[", r"Epoch\(train\) \[", r".*TrainEpoch-", r"INFO Train:.*", r".*\[Train\]\[Epoch.*", r".*images\/sec.*"] + combined_pattern = re.compile("|".join(epoch_pattern)) + epoch_matches = bool(combined_pattern.search(sout)) + + if matches: + # Get last match and convert to floats + last_acc1, last_acc5 = map(float, matches[-1]) + print(f"Acc@1: {last_acc1}, Acc@5: {last_acc5}") + result["result"].setdefault(prec, {"status": "PASS"}) + result["result"][prec]["acc1"] = last_acc1 + result["result"][prec]["acc5"] = last_acc5 + elif epoch_matches: + result["result"].setdefault(prec, {"status": "PASS"}) + result["result"][prec]["acc1"] = "train timeout" + result["result"][prec]["acc5"] = "train timeout" + else: + result["result"].setdefault(prec, {"status": "FAIL"}) + print("No match found.") + + result["result"][prec]["Cost time (s)"] = t + logging.debug(f"matchs:\n{matches}") + return result + +def get_metric_result(str): + if str: + return json.loads(str.replace("'", "\""))["metricResult"] + return None + +def run_script(script): + start_time = time.perf_counter() + result = subprocess.run( + script, shell=True, capture_output=True, text=True, executable="/bin/bash" + ) + end_time = time.perf_counter() + execution_time = end_time - start_time + logging.debug(f"执行命令：\n{script}") + logging.debug("执行时间: {:.4f} 秒".format(execution_time)) + logging.debug(f"标准输出: {result.stdout}") + logging.debug(f"标准错误: {result.stderr}") + logging.debug(f"返回码: {result.returncode}") + return result, execution_time + +if __name__ == "__main__": + # 配置日志 + debug_level = logging.DEBUG if is_debug() else logging.INFO + logging.basicConfig( + handlers=[logging.FileHandler("output.log"), logging.StreamHandler()], + level=debug_level, + format="%(asctime)s - %(levelname)s - %(message)s", + ) + main() -- Gitee