diff --git a/cv/classification/acmix/pytorch/ci/prepare.sh b/cv/classification/acmix/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..b7988b5384e39f48509a3de1da141168a090636e --- /dev/null +++ b/cv/classification/acmix/pytorch/ci/prepare.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +pip install termcolor==1.1.0 yacs==0.1.8 timm==0.4.5 +timeout 1800 bash run.sh 4 acmix_swin_small_patch4_window7_224.yaml ../imagenet \ No newline at end of file diff --git a/cv/classification/acnet/pytorch/ci/prepare.sh b/cv/classification/acnet/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..7154b51c217dd10ed18446c9c140db12e717cf6c --- /dev/null +++ b/cv/classification/acnet/pytorch/ci/prepare.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +git clone https://github.com/DingXiaoH/ACNet.git +cd ACNet +ln -s /mnt/deepspark/data/datasets/imagenet ./imagenet_data +rm -rf acnet/acb.py +rm -rf utils/misc.py +mv ../acb.py acnet/ +mv ../misc.py utils/ +export PYTHONPATH=$PYTHONPATH:. +# export CUDA_VISIBLE_DEVICES=0 +timeout 1800 python3 -m torch.distributed.launch --nproc_per_node=1 acnet/do_acnet.py -a sres18 -b acb --local_rank 0 +# export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +# timeout 1800 python3 -m torch.distributed.launch --nproc_per_node=8 acnet/do_acnet.py -a sres18 -b acb \ No newline at end of file diff --git a/cv/classification/alexnet/tensorflow/README.md b/cv/classification/alexnet/tensorflow/README.md index f95aad1c93dd899091697bc48b221d98080ba11e..59a8cb476769e2fda7d42932446516070514e0a6 100644 --- a/cv/classification/alexnet/tensorflow/README.md +++ b/cv/classification/alexnet/tensorflow/README.md @@ -57,4 +57,4 @@ bash run_train_alexnet_multigpu_imagenet.sh ## References -- [TensorFlow Models](https://github.com/tensorflow/models) +- [TensorFlow/benchmarks](https://github.com/tensorflow/benchmarks/tree/master/scripts/tf_cnn_benchmarks) diff --git a/cv/classification/cbam/pytorch/ci/prepare.sh b/cv/classification/cbam/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..463f53305a869e3a1fd3068ea9e5553813794138 --- /dev/null +++ b/cv/classification/cbam/pytorch/ci/prepare.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +timeout 1800 python3 train_imagenet.py --ngpu 4 --workers 20 --arch resnet --depth 50 --epochs 100 --batch-size 256 --lr 0.1 --att-type CBAM --prefix RESNET50_IMAGENET_CBAM ./imagenet \ No newline at end of file diff --git a/cv/classification/convnext/pytorch/ci/prepare.sh b/cv/classification/convnext/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..a4e7994462c541995d1dccd910cce3c44142cbec --- /dev/null +++ b/cv/classification/convnext/pytorch/ci/prepare.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +pip install timm==0.4.12 tensorboardX six torch torchvision + +git clone https://github.com/facebookresearch/ConvNeXt.git +cd ConvNeXt +git checkout 048efcea897d999aed302f2639b6270aedf8d4c8 +# fix torch._six import error +sed -i 's/from torch._six import inf/from torch import inf/g' utils.py +timeout 1800 python3 main.py --model convnext_tiny \ + --drop_path 0.1 \ + --batch_size 128 \ + --lr 4e-3 \ + --update_freq 4 \ + --model_ema true \ + --model_ema_eval true \ + --data_path ../imagenet \ + --output_dir ../save_results \ No newline at end of file diff --git a/cv/classification/densenet/paddlepaddle/README.md b/cv/classification/densenet/paddlepaddle/README.md index 678b4e196a5e65106609e9f6a4cbd392a96479a4..df4f0002461ba6adf074c78ff1d4e2f6134dd64d 100644 --- a/cv/classification/densenet/paddlepaddle/README.md +++ b/cv/classification/densenet/paddlepaddle/README.md @@ -34,16 +34,15 @@ imagenet ### Install Dependencies ```bash -git clone --recursive https://github.com/PaddlePaddle/PaddleClas.git - -cd PaddleClas/ - -yum install mesa-libGL -y - +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 +cd PaddleClas pip3 install -r requirements.txt -pip3 install protobuf==3.20.3 -pip3 install urllib3==1.26.13 - python3 setup.py install ``` diff --git a/cv/classification/efficientnet_b0/paddlepaddle/README.md b/cv/classification/efficientnet_b0/paddlepaddle/README.md index af8c0c9a39dc55a999f7e4aaedb2bc303f5bdceb..28416892241454903c6d050924233a8cdbfb8dc5 100644 --- a/cv/classification/efficientnet_b0/paddlepaddle/README.md +++ b/cv/classification/efficientnet_b0/paddlepaddle/README.md @@ -47,14 +47,16 @@ sed -i 's#^#val/#g' val_list.txt ### Install Dependencies ```bash +# Install libGL +## CentOS yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx -git clone -b release/2.5 https://github.com/PaddlePaddle/PaddleClas.git -cd PaddleClas/ +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 +cd PaddleClas pip3 install -r requirements.txt -pip3 install paddleclas -pip3 install protobuf==3.20.3 -pip3 install urllib3==1.26.15 +python3 setup.py install ``` diff --git a/cv/classification/fasternet/pytorch/ci/prepare.sh b/cv/classification/fasternet/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..88585352449fcbd84e7ce67f96d74c9eee19e119 --- /dev/null +++ b/cv/classification/fasternet/pytorch/ci/prepare.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +pip install -r requirements.txt +git clone https://github.com/JierunChen/FasterNet.git +cd FasterNet +git checkout e8fba4465ae912359c9f661a72b14e39347e4954 +timeout 1800 python3 train_test.py -g 0,1,2,3 --num_nodes 1 -n 4 -b 4096 -e 2000 \ + --data_dir ../imagenet \ + --pin_memory --wandb_project_name fasternet \ + --model_ckpt_dir ./model_ckpt/$(date +'%Y%m%d_%H%M%S') \ + --cfg cfg/fasternet_t0.yaml \ No newline at end of file diff --git a/cv/classification/googlenet/paddlepaddle/README.md b/cv/classification/googlenet/paddlepaddle/README.md index 6f4139d9c0d09962bfc3960c659d29eb77526cf9..61b77aba66ed6f661a33c79a8ea2cdcb6bf16630 100644 --- a/cv/classification/googlenet/paddlepaddle/README.md +++ b/cv/classification/googlenet/paddlepaddle/README.md @@ -34,9 +34,16 @@ imagenet ### Install Dependencies ```bash -git clone --recursive https://github.com/PaddlePaddle/PaddleClas.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 cd PaddleClas pip3 install -r requirements.txt +python3 setup.py install ``` ## Model Training @@ -61,3 +68,6 @@ export FLAGS_cudnn_batchnorm_spatial_persistent=True export CUDA_VISIBLE_DEVICES=0,1,2,3 python3 -u -m paddle.distributed.launch --gpus=0,1,2,3 tools/train.py -c ppcls/configs/ImageNet/Inception/GoogLeNet.yaml ``` + +## Reference +- [PaddleClas](https://github.com/PaddlePaddle/PaddleClas) \ No newline at end of file diff --git a/cv/classification/internimage/pytorch/ci/prepare.sh b/cv/classification/internimage/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..592cac8b8f6c57db93c00b866b4e5a38aa14eb9c --- /dev/null +++ b/cv/classification/internimage/pytorch/ci/prepare.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +## Install mmcv +cd mmcv/ +bash clean_mmcv.sh +bash build_mmcv.sh +bash install_mmcv.sh +cd ../ + +## Install timm and mmdet +pip3 install timm==0.6.11 mmdet==2.28.1 + +pip3 install addict yapf opencv-python termcolor yacs pyyaml scipy + +cd ./ops_dcnv3 +sh ./make.sh +# unit test (should see all checking is True) +# python3 test.py +cd ../ +# Training on 8 GPUs +# export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +# export LOCAL_SIZE=8 +# python3 -m torch.distributed.launch --nproc_per_node 8 --master_port 12345 main.py --cfg configs/internimage_t_1k_224.yaml --data-path ./imagenet + +# Training on 1 GPU +export CUDA_VISIBLE_DEVICES=0 +export LOCAL_SIZE=1 +timeout 1800 python3 main.py --cfg configs/internimage_t_1k_224.yaml --data-path ./imagenet \ No newline at end of file diff --git a/cv/classification/mobilenetv3/paddlepaddle/README.md b/cv/classification/mobilenetv3/paddlepaddle/README.md index 1b707203d434fd8cc9a031e60fb5a33bc106bc63..a0dd9fba3caf8a0fcfe0f12053af954427b525af 100644 --- a/cv/classification/mobilenetv3/paddlepaddle/README.md +++ b/cv/classification/mobilenetv3/paddlepaddle/README.md @@ -35,9 +35,18 @@ imagenet ### Install Dependencies ```bash -git clone https://github.com/PaddlePaddle/PaddleClas.git -cd PaddleClas/ +## Step 1: Installing +```bash +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 +cd PaddleClas pip3 install -r requirements.txt +python3 setup.py install ``` ## Model Training diff --git a/cv/classification/mobilenetv3_large_x1_0/paddlepaddle/README.md b/cv/classification/mobilenetv3_large_x1_0/paddlepaddle/README.md index 996ddf9f67162f6e85e020f0e481f6740f6caee3..f76bf582cad52cb93cdd44f90e92a25d748d838f 100644 --- a/cv/classification/mobilenetv3_large_x1_0/paddlepaddle/README.md +++ b/cv/classification/mobilenetv3_large_x1_0/paddlepaddle/README.md @@ -35,15 +35,17 @@ imagenet ### Install Dependencies ```bash -yum install mesa-libGL -y - -git clone https://github.com/PaddlePaddle/PaddleClas.git -cd PaddleClas/ +```bash +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 +cd PaddleClas pip3 install -r requirements.txt python3 setup.py install - -pip3 install protobuf==3.20.3 -pip3 install urllib3==1.26.13 ``` ## Model Training diff --git a/cv/classification/pp-lcnet/paddlepaddle/README.md b/cv/classification/pp-lcnet/paddlepaddle/README.md index 54decf1f039279ed81911bedfd050719a785dea5..04f937853bef5ab4ecba5309cd78668c6601d9c3 100644 --- a/cv/classification/pp-lcnet/paddlepaddle/README.md +++ b/cv/classification/pp-lcnet/paddlepaddle/README.md @@ -35,7 +35,13 @@ imagenet ### Install Dependencies ```bash -git clone --recursive https://github.com/PaddlePaddle/PaddleClas.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 cd PaddleClas pip3 install -r requirements.txt python3 setup.py install diff --git a/cv/classification/repmlp/pytorch/ci/prepare.sh b/cv/classification/repmlp/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..81b9928024e1da9fb0e4213f07627799eead00a9 --- /dev/null +++ b/cv/classification/repmlp/pytorch/ci/prepare.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +pip3 install timm yacs +git clone https://github.com/DingXiaoH/RepMLP.git +cd RepMLP +git checkout 3eff13fa0257af28663880d870f327d665f0a8e2 +# fix --local-rank for torch 2.x +sed -i 's/--local_rank/--local-rank/g' main_repmlp.py +# change dataset load +sed -i "s@dataset = torchvision.datasets.ImageNet(root=config.DATA.DATA_PATH, split='train' if is_train else 'val', transform=transform)@dataset = datasets.ImageFolder(os.path.join(config.DATA.DATA_PATH, prefix), transform=transform)@" data/build.py + +timeout 1800 python3 -m torch.distributed.launch --nproc_per_node 4 --master_port 12349 main_repmlp.py --arch RepMLPNet-B256 --batch-size 32 --tag my_experiment --opts TRAIN.EPOCHS 100 TRAIN.BASE_LR 0.001 TRAIN.WEIGHT_DECAY 0.1 TRAIN.OPTIMIZER.NAME adamw TRAIN.OPTIMIZER.MOMENTUM 0.9 TRAIN.WARMUP_LR 5e-7 TRAIN.MIN_LR 0.0 TRAIN.WARMUP_EPOCHS 10 AUG.PRESET raug15 AUG.MIXUP 0.4 AUG.CUTMIX 1.0 DATA.IMG_SIZE 256 --data-path ../imagenet \ No newline at end of file diff --git a/cv/classification/repvgg/pytorch/ci/prepare.sh b/cv/classification/repvgg/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..b97a6abe45fa0370bbb9f41b1d104610c19ccf8f --- /dev/null +++ b/cv/classification/repvgg/pytorch/ci/prepare.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +pip3 install timm yacs +git clone https://github.com/DingXiaoH/RepVGG.git +cd RepVGG +git checkout eae7c5204001eaf195bbe2ee72fb6a37855cce33 +# fix --local-rank for torch 2.x +sed -i 's/--local_rank/--local-rank/g' main.py + +# change dataset load +# Tips: "import os" into data/build.py +sed -i "s@dataset = torchvision.datasets.ImageNet(root=config.DATA.DATA_PATH, split='train' if is_train else 'val', transform=transform)@dataset = datasets.ImageFolder(os.path.join(config.DATA.DATA_PATH, prefix), transform=transform)@" data/build.py + +timeout 1800 python3 -m torch.distributed.launch --nproc_per_node 4 --master_port 12349 main.py --arch RepVGG-A0 --data-path ../imagenet --batch-size 32 --tag train_from_scratch --output ./ --opts TRAIN.EPOCHS 300 TRAIN.BASE_LR 0.1 TRAIN.WEIGHT_DECAY 1e-4 TRAIN.WARMUP_EPOCHS 5 MODEL.LABEL_SMOOTHING 0.1 AUG.PRESET weak AUG.MIXUP 0.0 DATA.DATASET imagenet DATA.IMG_SIZE 224 \ No newline at end of file diff --git a/cv/classification/repvit/pytorch/ci/prepare.sh b/cv/classification/repvit/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..6b5940a82f45805263c7a5bd2bdee59579db82a9 --- /dev/null +++ b/cv/classification/repvit/pytorch/ci/prepare.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +git clone https://github.com/THU-MIG/RepViT.git +cd RepViT +git checkout 298f42075eda5d2e6102559fad260c970769d34e +pip3 install -r requirements.txt +# On single GPU +timeout 1800 python3 main.py --model repvit_m0_9 --data-path ../imagenet --dist-eval + +# # Multiple GPUs on one machine +# python3 -m torch.distributed.launch --nproc_per_node=8 --master_port 12346 --use_env main.py --model repvit_m0_9 --data-path ./imagenet --dist-eval \ No newline at end of file diff --git a/cv/classification/res2net50_14w_8s/paddlepaddle/README.md b/cv/classification/res2net50_14w_8s/paddlepaddle/README.md index c34af020488cb3e817683afa19fc9ea5fe2822bd..c19b2c642274ccb27f0b80073b5492d3a9a91b6d 100644 --- a/cv/classification/res2net50_14w_8s/paddlepaddle/README.md +++ b/cv/classification/res2net50_14w_8s/paddlepaddle/README.md @@ -34,11 +34,16 @@ imagenet ### Install Dependencies ```bash -git clone -b release/2.5 https://github.com/PaddlePaddle/PaddleClas.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 cd PaddleClas pip3 install -r requirements.txt -pip3 install protobuf==3.20.3 urllib3==1.26.6 -yum install -y mesa-libGL +python3 setup.py install ``` ## Model Training diff --git a/cv/classification/resnest50/paddlepaddle/README.md b/cv/classification/resnest50/paddlepaddle/README.md index 9717d0caa703594d3d672a605cbc3b0dfc285ef1..0fae57eefb1e2b64223f3a3f2f624e34d79326f7 100644 --- a/cv/classification/resnest50/paddlepaddle/README.md +++ b/cv/classification/resnest50/paddlepaddle/README.md @@ -35,9 +35,16 @@ imagenet ### Install Dependencies ```bash -git clone --recursive https://github.com/PaddlePaddle/PaddleClas.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 cd PaddleClas pip3 install -r requirements.txt +python3 setup.py install ``` ## Model Training @@ -55,3 +62,6 @@ python3 -u -m paddle.distributed.launch --gpus=0,1,2,3 tools/train.py -c ppcls/c | Model | GPU | FP32 | |-----------|------------|--------------| | ResNeSt50 | BI-V100 x4 | Acc@1=0.7677 | + +## Reference +- [PaddleClas](https://github.com/PaddlePaddle/PaddleClas) \ No newline at end of file diff --git a/cv/classification/resnet50/paddlepaddle/README.md b/cv/classification/resnet50/paddlepaddle/README.md index 5151cfb89168c1901ed7dcb0571770622cab2cb8..5e029d807d0f6bc645f5953dbf9aefed3296b716 100644 --- a/cv/classification/resnet50/paddlepaddle/README.md +++ b/cv/classification/resnet50/paddlepaddle/README.md @@ -34,12 +34,16 @@ imagenet ### Install Dependencies ```bash -git clone --recursive https://github.com/PaddlePaddle/PaddleClas.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 cd PaddleClas pip3 install -r requirements.txt -yum install mesa-libGL -y -pip3 install urllib3==1.26.6 -pip3 install protobuf==3.20.3 +python3 setup.py install ``` Tips: for `PaddleClas` training, the images path in train_list.txt and val_list.txt must contain `train/` and `val/` directories: @@ -71,3 +75,6 @@ python3 -u -m paddle.distributed.launch --gpus=0,1,2,3 tools/train.py -c ppcls/c | Model | GPU | FP32 | |----------|------------|------------------------------------| | ResNet50 | BI-V100 x4 | Acc@1=76.27,FPS=80.37,BatchSize=64 | + +## Reference +- [PaddleClas](https://github.com/PaddlePaddle/PaddleClas) \ No newline at end of file diff --git a/cv/classification/resnet50/tensorflow/README.md b/cv/classification/resnet50/tensorflow/README.md index c12d7e5e358ecdc182f47d24543ddf65617917d1..3832bc54e01ac7a5f1800a824c05ce2de0866d15 100644 --- a/cv/classification/resnet50/tensorflow/README.md +++ b/cv/classification/resnet50/tensorflow/README.md @@ -39,3 +39,6 @@ bash run_train_resnet50_multigpu_imagenette.sh | Model | GPU | acc | fps | |----------|------------|--------|-------| | ResNet50 | BI-V100 x8 | 0.9860 | 236.9 | + +## Reference +- [TensorFlow/benchmarks](https://github.com/tensorflow/benchmarks/tree/master/scripts/tf_cnn_benchmarks) \ No newline at end of file diff --git a/cv/classification/resnet50/tensorflow/run_train_resnet50_imagenette.sh b/cv/classification/resnet50/tensorflow/run_train_resnet50_imagenette.sh index 171fd87614545938c794da5427336364415571a4..a8a17884446b99e4cd6737375d3b6ac3d06d9079 100644 --- a/cv/classification/resnet50/tensorflow/run_train_resnet50_imagenette.sh +++ b/cv/classification/resnet50/tensorflow/run_train_resnet50_imagenette.sh @@ -40,7 +40,7 @@ do done python3 -u tf_cnn_benchmarks.py\ - --data_name=imagenette --data_dir=${DATA_DIR}\ + --data_name=imagenet --data_dir=${DATA_DIR}\ --data_format=NCHW --batch_size=${BATCH_SIZE}\ --model=resnet50 --optimizer=${OPTIMIZER} --num_gpus=1\ --weight_decay=1e-4 --train_dir=${TRAIN_DIR}\ diff --git a/cv/classification/resnet50/tensorflow/run_train_resnet50_multigpu_imagenette.sh b/cv/classification/resnet50/tensorflow/run_train_resnet50_multigpu_imagenette.sh index 1a2c33ad7ab84a1078f9eeaeb94522c72591e797..c4ab33fa3f8e840c4d75e295c9069242eed33efb 100644 --- a/cv/classification/resnet50/tensorflow/run_train_resnet50_multigpu_imagenette.sh +++ b/cv/classification/resnet50/tensorflow/run_train_resnet50_multigpu_imagenette.sh @@ -42,7 +42,7 @@ done source ./get_num_devices.sh UMD_WAITAFTERLAUNCH=1 python3 -u tf_cnn_benchmarks.py\ - --data_name=imagenette --data_dir=${DATA_DIR}\ + --data_name=imagenet --data_dir=${DATA_DIR}\ --data_format=NCHW --batch_size=${BATCH_SIZE}\ --model=resnet50 --optimizer=${OPTIMIZER} --num_gpus=${IX_NUM_CUDA_VISIBLE_DEVICES}\ --weight_decay=1e-4 --train_dir=${TRAIN_DIR}\ diff --git a/cv/classification/se_resnet50_vd/paddlepaddle/README.md b/cv/classification/se_resnet50_vd/paddlepaddle/README.md index 9f74f52cbb4621ddb66d4d2877130d647be5a7e2..88ae57427efde7b63b5fcd84de436d791ba068ca 100644 --- a/cv/classification/se_resnet50_vd/paddlepaddle/README.md +++ b/cv/classification/se_resnet50_vd/paddlepaddle/README.md @@ -46,17 +46,24 @@ sed -i 's#^#val/#g' val_list.txt ### Install Dependencies ```bash -pip3 install -r requirements.txt -python3 -m pip install urllib3==1.26.6 +# Install libGL +## CentOS yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx -git clone -b release/2.5 https://github.com/PaddlePaddle/PaddleClas.git +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 +cd PaddleClas +pip3 install -r requirements.txt +python3 setup.py install ``` ## Model Training ```bash cd PaddleClas/ +# Link your dataset to the default location +ln -s /path/to/imagenet ./dataset/ILSVRC2012 export CUDA_VISIBLE_DEVICES=0,1,2,3 python3 -m paddle.distributed.launch --gpus="0,1,2,3" tools/train.py -c ./ppcls/configs/ImageNet/SENet/SE_ResNet50_vd.yaml ``` @@ -69,4 +76,4 @@ python3 -m paddle.distributed.launch --gpus="0,1,2,3" tools/train.py -c ./ppcls/ ## References -- [PaddleClas](https://github.com/PaddlePaddle/PaddleClas/tree/release/2.5) +- [PaddleClas](https://github.com/PaddlePaddle/PaddleClas) diff --git a/cv/classification/se_resnet50_vd/paddlepaddle/requirements.txt b/cv/classification/se_resnet50_vd/paddlepaddle/requirements.txt deleted file mode 100644 index 4d1ad019dce7b6393e1a4d665018e64348777266..0000000000000000000000000000000000000000 --- a/cv/classification/se_resnet50_vd/paddlepaddle/requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ -prettytable -ujson -opencv-python==4.6.0.66 -pillow -tqdm -PyYAML>=5.1 -visualdl>=2.2.0 -scipy>=1.0.0 -scikit-learn==1.0.2 -gast==0.3.3 -faiss-cpu -easydict -visualdl==2.4.0 \ No newline at end of file diff --git a/cv/classification/shufflenetv2/paddlepaddle/README.md b/cv/classification/shufflenetv2/paddlepaddle/README.md index 65bdbbf9523ab99c3c4c32a095e671afd810004f..00d024039a77a5eab63461249dbe6f00231339bc 100644 --- a/cv/classification/shufflenetv2/paddlepaddle/README.md +++ b/cv/classification/shufflenetv2/paddlepaddle/README.md @@ -34,16 +34,17 @@ imagenet ### Install Dependencies ```bash +# Install libGL +## CentOS yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx -git clone --recursive https://github.com/PaddlePaddle/PaddleClas.git -cd PaddleClas/ +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 +cd PaddleClas pip3 install -r requirements.txt python3 setup.py install -pip3 install protobuf==3.20.3 -pip3 install urllib3==1.26.13 - ``` ## Model Training diff --git a/cv/classification/swin_transformer/paddlepaddle/README.md b/cv/classification/swin_transformer/paddlepaddle/README.md index d63622abf20367cdda92fde8803b1b6b23ede6e2..dda047a8de7ab86ebbe55b569d277763fd82bc44 100644 --- a/cv/classification/swin_transformer/paddlepaddle/README.md +++ b/cv/classification/swin_transformer/paddlepaddle/README.md @@ -35,9 +35,16 @@ imagenet ### Install Dependencies ```bash -git clone --recursive https://github.com/PaddlePaddle/PaddleClas.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 cd PaddleClas pip3 install -r requirements.txt +python3 setup.py install ``` ## Model Training @@ -57,3 +64,7 @@ python3 -u -m paddle.distributed.launch --gpus=0,1,2,3,4,5,6,7 tools/train.py -c | Model | GPU | FP32 | |------------------|------------|--------------| | Swin Transformer | BI-V100 x8 | Acc@1=0.8024 | + +## References + +- [PaddleClas](https://github.com/PaddlePaddle/PaddleClas) \ No newline at end of file diff --git a/cv/classification/swin_transformer/pytorch/ci/prepare.sh b/cv/classification/swin_transformer/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..330fd94a9624c783af2d64e2106ffd698c85907e --- /dev/null +++ b/cv/classification/swin_transformer/pytorch/ci/prepare.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +git clone https://github.com/microsoft/Swin-Transformer.git +git checkout f82860bfb5225915aca09c3227159ee9e1df874d +cd Swin-Transformer +pip install timm==0.4.12 yacs +## fix --local-rank for torch 2.x +sed -i 's/--local_rank/--local-rank/g' main.py + +timeout 1800 python3 -m torch.distributed.launch --nproc_per_node 4 --master_port 12345 main.py \ + --cfg configs/swin/swin_tiny_patch4_window7_224.yaml --data-path ../imagenet --batch-size 128 \ No newline at end of file diff --git a/cv/classification/vgg/paddlepaddle/README.md b/cv/classification/vgg/paddlepaddle/README.md index df26ef1909715f6a634d42b9dbd4f3e728318e37..44829679ee27e8e12e88318f0c9a488e02a1d70f 100644 --- a/cv/classification/vgg/paddlepaddle/README.md +++ b/cv/classification/vgg/paddlepaddle/README.md @@ -34,12 +34,16 @@ imagenet ### Install Dependencies ```bash -git clone https://github.com/PaddlePaddle/PaddleClas.git -``` +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx -```bash +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 cd PaddleClas pip3 install -r requirements.txt +python3 setup.py install ``` ## Model Training diff --git a/cv/classification/vgg/tensorflow/run_train_vgg16_imagenet.sh b/cv/classification/vgg/tensorflow/run_train_vgg16_imagenet.sh index 342a88f67397739aacd3ca438fea87ffe1446449..aece8bfe02ab8ec1c312dcff9b98cb139f8207ea 100644 --- a/cv/classification/vgg/tensorflow/run_train_vgg16_imagenet.sh +++ b/cv/classification/vgg/tensorflow/run_train_vgg16_imagenet.sh @@ -17,7 +17,7 @@ export TF_CUDNN_USE_AUTOTUNE=1 export TF_CPP_MIN_LOG_LEVEL=1 -: ${BATCH_SIZE:=256} +: ${BATCH_SIZE:=32} #TRAIN_EPOCHS=10 # optional optimizer: adam, rmsprop, momentum, sgd OPTIMIZER=momentum diff --git a/cv/classification/wavemlp/pytorch/ci/prepare.sh b/cv/classification/wavemlp/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..79daef5fbe10bb182684a66f42b7b0edaba5db6f --- /dev/null +++ b/cv/classification/wavemlp/pytorch/ci/prepare.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +pip install thop timm==0.4.5 torchprofile +git clone https://github.com/huawei-noah/Efficient-AI-Backbones.git +cd Efficient-AI-Backbones/wavemlp_pytorch/ +git checkout 25531f7fdcf61e300b47c52ba80973d0af8bb011 + +# fix --local-rank for torch 2.x +sed -i 's/--local_rank/--local-rank/g' train.py +# change dataset +sed -i "s@from timm.data import Dataset@from timm.data import ImageDataset@" train.py +sed -i "s@dataset_train = Dataset(train_dir)@dataset_train = ImageDataset(train_dir)@" train.py +sed -i "s@dataset_eval = Dataset(eval_dir)@dataset_eval = ImageDataset(eval_dir)@" train.py +sed -i 's/args.max_history/100/g' train.py + +python3 -m torch.distributed.launch --nproc_per_node 8 --nnodes=1 --node_rank=0 train.py ../imagenet/ --output ./output/ --model WaveMLP_T_dw --sched cosine --epochs 300 --opt adamw -j 8 --warmup-lr 1e-6 --mixup .8 --cutmix 1.0 --model-ema --model-ema-decay 0.99996 --aa rand-m9-mstd0.5-inc1 --color-jitter 0.4 --warmup-epochs 5 --opt-eps 1e-8 --repeated-aug --remode pixel --reprob 0.25 --amp --lr 1e-3 --weight-decay .05 --drop 0 --drop-path 0.1 -b 128 \ No newline at end of file diff --git a/cv/classification/xception/paddlepaddle/README.md b/cv/classification/xception/paddlepaddle/README.md index 425e10787afa8ff8a344e141e24baea58d30b80d..c31dd9ef95113b0ac0a3ec101526dbf0ffd33356 100644 --- a/cv/classification/xception/paddlepaddle/README.md +++ b/cv/classification/xception/paddlepaddle/README.md @@ -47,10 +47,16 @@ sed -i 's#^#val/#g' val_list.txt ### Install Dependencies ```bash -git clone -b release/2.5 https://github.com/PaddlePaddle/PaddleClas.git -cd PaddleClas -pip3 install scikit-learn easydict visualdl==2.2.0 urllib3==1.26.6 +# Install libGL +## CentOS yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleClas.git -b release/2.6 --depth=1 +cd PaddleClas +pip3 install -r requirements.txt +python3 setup.py install ``` ## Model Training diff --git a/cv/detection/centermask2/pytorch/README.md b/cv/detection/centermask2/pytorch/README.md index 6992d06982e4e2353e9d42aa7dc7b1388bee3c8a..4d5d8e3ce3750c1abe3e6ea7130cff5f3b812758 100644 --- a/cv/detection/centermask2/pytorch/README.md +++ b/cv/detection/centermask2/pytorch/README.md @@ -61,7 +61,7 @@ one should execute: ```bash git clone https://github.com/youngwanLEE/centermask2.git cd centermask2 -python3 train_net.py --config-file "configs/centermask/centermask_V_39_eSE_FPN_ms_3x.yaml" --num-gpus 8 +python3 train_net.py --config-file configs/centermask/centermask_R_50_FPN_ms_3x.yaml --num-gpus 8 ``` ## References diff --git a/cv/detection/centermask2/pytorch/ci/prepare.sh b/cv/detection/centermask2/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..419cbc32759ff10700138fb35100ee03b26c6926 --- /dev/null +++ b/cv/detection/centermask2/pytorch/ci/prepare.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +# Install detectron2 with 9604f5995cc628619f0e4fd913453b4d7d61db3f +git clone https://github.com/facebookresearch/detectron2.git +python3 -m pip install -e detectron2 + +# 6adabd66034347b7da07f2d474e4baa1c27b54ee is the commit hash of centermask2 +git clone https://github.com/youngwanLEE/centermask2.git +cd centermask2 +# fix RuntimeError: "max_elementwise_cuda" not implemented for 'Double' +sed -i 's@torch.max(mask_ratios, value_eps)@torch.max(mask_ratios.cpu(), value_eps.cpu()).cuda()@' centermask/modeling/centermask/mask_head.py +sed -i 's@torch.max(mask_union_area, value_1)@torch.max(mask_union_area.cpu(), value_1.cpu()).cuda()@' centermask/modeling/centermask/mask_head.py +sed -i 's@torch.max(mask_ovr_area, value_0)@torch.max(mask_ovr_area.cpu(), value_0.cpu()).cuda()@' centermask/modeling/centermask/mask_head.py + +mkdir -p datasets/ +ln -s /mnt/deepspark/data/datasets/coco2017 datasets/coco +mkdir -p /root/.torch/iopath_cache/detectron2/ImageNetPretrained/MSRA/ +cp /mnt/deepspark/data/checkpoints/R-50.pkl /root/.torch/iopath_cache/detectron2/ImageNetPretrained/MSRA/ +timeout 1800 python3 train_net.py --config-file configs/centermask/centermask_R_50_FPN_ms_3x.yaml --num-gpus 4 \ No newline at end of file diff --git a/cv/detection/centernet/paddlepaddle/README.md b/cv/detection/centernet/paddlepaddle/README.md index f1ad9a71b7b8d5855f5fd167c26b4bb3a777d403..afec74dc023590747d9b83cd253e9d66eea0d23e 100644 --- a/cv/detection/centernet/paddlepaddle/README.md +++ b/cv/detection/centernet/paddlepaddle/README.md @@ -13,7 +13,13 @@ can be extended to 3D detection and pose estimation tasks. ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset @@ -52,3 +58,6 @@ python3 -m paddle.distributed.launch --gpus 0,1,2,3,4,5,6,7 tools/train.py -c co | GPU | learning rate | FPS | Train Epochs | mAP | |------------|---------------|-------|--------------|------| | BI-V100 x8 | 0.00005 | 10.85 | 3 | 38.5 | + +## Reference +[PaddleDetection](https://github.com/PaddlePaddle/PaddleDetection) \ No newline at end of file diff --git a/cv/detection/centernet/pytorch/README.md b/cv/detection/centernet/pytorch/README.md index 2b97bd94229e483030cb50f72241f0c0bf1a882a..dd0198cf5547007b95afc8b73307347a275e066a 100644 --- a/cv/detection/centernet/pytorch/README.md +++ b/cv/detection/centernet/pytorch/README.md @@ -72,7 +72,7 @@ python3 setup.py build develop ## Model Training ```bash -cd ./cv/detection/centernet/pytorch/src +cd ./src touch lib/datasets/__init__.py export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 diff --git a/cv/detection/centernet/pytorch/ci/prepare.sh b/cv/detection/centernet/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..89047cc83ecd7a1e87d756ec807e56be6db16cbe --- /dev/null +++ b/cv/detection/centernet/pytorch/ci/prepare.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +pip3 install -r requirements.txt +git clone https://github.com/xingyizhou/CenterNet.git +cd CenterNet +git checkout 4c50fd3a46bdf63dbf2082c5cbb3458d39579e6c +# Compile deformable convolutional(DCNv2) +cd ./src/lib/models/networks/ +rm -rf DCNv2 +git clone -b pytorch_1.11 https://github.com/lbin/DCNv2.git +cd ./DCNv2/ +python3 setup.py build develop +cd ../../../../../ +ln -s /mnt/deepspark/data/datasets/coco2017 ./data/coco +mkdir -p /root/.cache/torch/hub/checkpoints/ +cp /mnt/deepspark/data/checkpoints/resnet18-5c106cde.pth /root/.cache/torch/hub/checkpoints/ + +cd ./src +touch lib/datasets/__init__.py +timeout 1800 python3 main.py ctdet --arch res_18 --batch_size 32 --master_batch 15 --lr 1.25e-4 --gpus 0 \ No newline at end of file diff --git a/cv/detection/co-detr/pytorch/README.md b/cv/detection/co-detr/pytorch/README.md index c79785662da66eaa06b570a2fb0436e208a371f5..b73ec656bfebf59748348441713e85b104da4008 100644 --- a/cv/detection/co-detr/pytorch/README.md +++ b/cv/detection/co-detr/pytorch/README.md @@ -51,10 +51,6 @@ git clone https://github.com/open-mmlab/mmdetection.git -b v3.3.0 --depth=1 cd mmdetection pip install -v -e . -# Install requirements -pip3 install -r requirements.txt -pip3 install urllib3==1.26.15 - # Download repo git clone https://github.com/Sense-X/Co-DETR.git cd /path/to/Co-DETR @@ -72,12 +68,9 @@ ln -s /path/to/coco ./data export CUDA_VISIBLE_DEVICES=0 python3 tools/train.py projects/configs/co_deformable_detr/co_deformable_detr_r50_1x_coco.py --work-dir path_to_exp --no-validate --auto-resume -# Eight GPUs -bash tools/dist_train.sh projects/configs/co_deformable_detr/co_deformable_detr_r50_1x_coco.py 8 path_to_exp --no-validate --auto-resume - -# Evaluation -export CUDA_VISIBLE_DEVICES=0 -PYTHONPATH=".:$PYTHONPATH" python3 tools/test.py projects/configs/co_deformable_detr/co_deformable_detr_r50_1x_coco.py path_to_exp/latest.pth --eval bbox +sed -i 's/python /python3 /g' tools/dist_train.sh +# Multiple GPUs on one machine +bash tools/dist_train.sh projects/CO-DETR/configs/codino/co_dino_5scale_r50_lsj_8xb2_1x_coco.py 8 ``` ## Model Results @@ -86,7 +79,5 @@ PYTHONPATH=".:$PYTHONPATH" python3 tools/test.py projects/configs/co_deformable_ |---------|------------|------|--------------|--------| | Co-DETR | BI-V100 x8 | 9.02 | 12 | 0.428 | -## References - -- [Paper](https://arxiv.org/pdf/2211.12860.pdf) -- [Co-DETR](https://github.com/Sense-X/Co-DETR) +## Reference +[mmdetection](https://github.com/open-mmlab/mmdetection) diff --git a/cv/detection/co-detr/pytorch/requirements.txt b/cv/detection/co-detr/pytorch/requirements.txt deleted file mode 100644 index 70e3bb137ad680daec449e0df0b8559fd1fb30a4..0000000000000000000000000000000000000000 --- a/cv/detection/co-detr/pytorch/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -addict -pycocotools -terminaltables -yapf==0.40.1 -fairscale -timm diff --git a/cv/detection/cornernet_mmdet/pytorch/README.md b/cv/detection/cornernet_mmdet/pytorch/README.md index 169c9c8ce7abbbceeb2a99cb1f8f508b7a3d6338..ffaf9595706421e0d74ec2914ad9d261796a23ea 100644 --- a/cv/detection/cornernet_mmdet/pytorch/README.md +++ b/cv/detection/cornernet_mmdet/pytorch/README.md @@ -78,6 +78,5 @@ bash tools/dist_train.sh configs/cornernet/cornernet_hourglass104_8xb6-210e-mste |-----------|------------|----------| | CornerNet | BI-V100 x8 | MAP=41.2 | -## References - -- [Paper](https://arxiv.org/abs/1808.01244) +## Reference +[mmdetection](https://github.com/open-mmlab/mmdetection) diff --git a/cv/detection/dcnv2_mmdet/pytorch/README.md b/cv/detection/dcnv2_mmdet/pytorch/README.md index 4f7f8595ea25439a5e396245c6b16a819afa6f02..8977ca8f0a46a1aaedfa30b513d136f3757e2aff 100644 --- a/cv/detection/dcnv2_mmdet/pytorch/README.md +++ b/cv/detection/dcnv2_mmdet/pytorch/README.md @@ -78,6 +78,5 @@ bash tools/dist_train.sh configs/dcnv2/faster-rcnn_r50-mdconv-c3-c5_fpn_1x_coco. |-------|------------|------| | DCNv2 | BI-V100 x8 | 41.2 | -## References - -- [Paper](https://arxiv.org/abs/1811.11168) +## Reference +[mmdetection](https://github.com/open-mmlab/mmdetection) diff --git a/cv/detection/detr/paddlepaddle/README.md b/cv/detection/detr/paddlepaddle/README.md index 326b1b5acd4ada485200c4590527231063e25282..701b4f8b11a23fade77bbb6f6c1555b3bafef5bb 100644 --- a/cv/detection/detr/paddlepaddle/README.md +++ b/cv/detection/detr/paddlepaddle/README.md @@ -13,7 +13,13 @@ while achieving competitive performance on benchmarks like COCO, offering a new ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset @@ -52,3 +58,7 @@ python3 -m paddle.distributed.launch --gpus 0,1,2,3,4,5,6,7 tools/train.py -c co | Model | GPU | learning rate | FPS | Train Epochs | Box AP | |-------|------------|---------------|-------|--------------|--------| | DETR | BI-V100 x8 | 0.00001 | 14.64 | 1 | 42.0 | + +## Reference + +- [PaddleDetection](https://github.com/PaddlePaddle/PaddleDetection) \ No newline at end of file diff --git a/cv/detection/fasterrcnn/pytorch/ci/prepare.sh b/cv/detection/fasterrcnn/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..1a769f5cb23326b9f5292c3c60c94b788e836554 --- /dev/null +++ b/cv/detection/fasterrcnn/pytorch/ci/prepare.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +mkdir -p data/ +ln -s /mnt/deepspark/data/datasets/coco2017 data/coco +mkdir -p /root/.cache/torch/hub/checkpoints/ +cp /mnt/deepspark/data/checkpoints/resnet50-0676ba61.pth /root/.cache/torch/hub/checkpoints/ + +cd start_scripts +bash init_torch.sh + +timeout 1800 bash train_fasterrcnn_resnet50_amp_torch.sh --dataset coco --data-path ../data/coco + +# bash train_fasterrcnn_resnet50_amp_dist_torch.sh --dataset coco --data-path ../data/coco diff --git a/cv/detection/fcos/paddlepaddle/README.md b/cv/detection/fcos/paddlepaddle/README.md index 877b88a6ecdf4d5c29bac39453a3b903bec17c73..bede6c790c6dfb3a9f3d494b072291aa73b4df83 100644 --- a/cv/detection/fcos/paddlepaddle/README.md +++ b/cv/detection/fcos/paddlepaddle/README.md @@ -13,7 +13,13 @@ it efficient and effective for various detection tasks. ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset @@ -47,3 +53,7 @@ python3 tools/train.py -c configs/fcos/fcos_r50_fpn_1x_coco.yml --eval | Model | GPU | FPS | Train Epochs | Box AP | |-------|------------|------|--------------|--------| | FCOS | BI-V100 x8 | 8.24 | 12 | 39.7 | + +## Reference + +- [PaddleDetection](https://github.com/PaddlePaddle/PaddleDetection) \ No newline at end of file diff --git a/cv/detection/fcos/pytorch/ci/prepare.sh b/cv/detection/fcos/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..da77dbf33f824f94f22eb6961c12af73b803af2f --- /dev/null +++ b/cv/detection/fcos/pytorch/ci/prepare.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +pip3 install -r requirements.txt +pip uninstall numpy +pip install numpy==1.23.5 +python3 setup.py develop + +mkdir -p data +ln -s /mnt/deepspark/data/datasets/coco2017 data/coco +mkdir -p /root/.torch/models/ +cp /mnt/deepspark/data/checkpoints/R-50.pkl /root/.torch/models/ + +timeout 1800 python3 -m torch.distributed.launch \ + --nproc_per_node=4 \ + tools/train_net.py \ + --config-file configs/fcos/fcos_imprv_R_50_FPN_1x.yaml \ + DATALOADER.NUM_WORKERS 2 \ + OUTPUT_DIR training_dir/fcos_imprv_R_50_FPN_1x \ No newline at end of file diff --git a/cv/detection/mamba_yolo/pytorch/README.md b/cv/detection/mamba_yolo/pytorch/README.md index 29d77994172c687b62448c6501d16805bb05fb1e..be6a2484bfa4baae6bab8373b05f0df685e20124 100644 --- a/cv/detection/mamba_yolo/pytorch/README.md +++ b/cv/detection/mamba_yolo/pytorch/README.md @@ -49,9 +49,8 @@ vim ultralytics/cfg/datasets/coco.yaml ```sh pip3 install seaborn thop timm einops - git clone --depth 1 https://gitee.com/deep-spark/deepsparkhub-GPL.git -cd cv/detection/mamba-yolo/pytorch +cd deepsparkhub-GPL/cv/detection/mamba-yolo/pytorch cd selective_scan && pip install . && cd .. pip install -v -e . diff --git a/cv/detection/mamba_yolo/pytorch/ci/prepare.sh b/cv/detection/mamba_yolo/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..81c889492644aaf094d6397ec4984a2e673b5875 --- /dev/null +++ b/cv/detection/mamba_yolo/pytorch/ci/prepare.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +pip3 install seaborn thop timm einops + +mkdir -p /root/.config/Ultralytics/ +cp /mnt/deepspark/data/3rd_party/Arial.ttf /root/.config/Ultralytics/ + +git clone --depth 1 https://gitee.com/deep-spark/deepsparkhub-GPL.git +cd deepsparkhub-GPL/cv/detection/mamba-yolo/pytorch + +cd selective_scan && pip install . && cd .. +pip install -v -e . + +mkdir -p /root/data +ln -s /mnt/deepspark/data/datasets/coco /root/data/coco +sed -i 's/\/mnt\/datasets\/MSCOCO2017/\/root\/data\/coco/g' ultralytics/cfg/datasets/coco.yaml +ln -s /mnt/deepspark/data/checkpoints/yolov8n.pt ./ + +timeout 1800 python3 mbyolo_train.py --task train --data ultralytics/cfg/datasets/coco.yaml \ + --config ultralytics/cfg/models/mamba-yolo/Mamba-YOLO-T.yaml \ +--amp --project ./output_dir/mscoco --name mambayolo_n --device 0,1,2,3 \ No newline at end of file diff --git a/cv/detection/maskrcnn/paddlepaddle/README.md b/cv/detection/maskrcnn/paddlepaddle/README.md index 7efd041385ca1ce3afbdac8deb22b7c115d2d8dd..f98a7cd692c38113ec1cae52ecadffae230890c1 100644 --- a/cv/detection/maskrcnn/paddlepaddle/README.md +++ b/cv/detection/maskrcnn/paddlepaddle/README.md @@ -13,11 +13,20 @@ detection, instance segmentation, and human pose estimation. ### Prepare Resources ```bash +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone --recursive https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 git clone https://github.com/PaddlePaddle/PaddleDetection.git cd PaddleDetection/ # Get COCO Dataset python3 dataset/coco/download_coco.py +or +ln -s /path/to/coco2017 dataset/coco ``` ### Install Dependencies @@ -42,3 +51,7 @@ python3 -u -m paddle.distributed.launch --gpus 0,1,2,3,4,5,6,7 tools/train.py -c | Model | GPU | FP32 | |------------|------------|-------------------------------| | Mask R-CNN | BI-V100 x8 | bbox=38.8,FPS=7.5,BatchSize=1 | + +## Reference + +- [PaddleDetection](https://github.com/PaddlePaddle/PaddleDetection) \ No newline at end of file diff --git a/cv/detection/maskrcnn/pytorch/ci/prepare.sh b/cv/detection/maskrcnn/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..b2887561321ead6a2b73d35651c5e17e1ec9d72b --- /dev/null +++ b/cv/detection/maskrcnn/pytorch/ci/prepare.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +pip3 install -r requirements.txt + +mkdir -p ./datasets/ +ln -s /mnt/deepspark/data/datasets/coco2017 ./datasets/coco +mkdir -p /root/.cache/torch/hub/checkpoints/ +cp /mnt/deepspark/data/checkpoints/resnet50-0676ba61.pth /root/.cache/torch/hub/checkpoints/resnet50-0676ba61.pth + +### Single Card +timeout 1800 python3 train.py --data-path ./datasets/coco --dataset coco --model maskrcnn_resnet50_fpn --lr 0.001 --batch-size 4 + +### AMP +# python3 train.py --data-path ./datasets/coco --dataset coco --model maskrcnn_resnet50_fpn --lr 0.001 --batch-size 1 --amp + +### DDP + +# python3 -m torch.distributed.launch --nproc_per_node=8 --use_env train.py\ + # --data-path ./datasets/coco --dataset coco --model maskrcnn_resnet50_fpn --wd 0.000001 --lr 0.001 --batch-size 4 \ No newline at end of file diff --git a/cv/detection/oc_sort/paddlepaddle/README.md b/cv/detection/oc_sort/paddlepaddle/README.md index 3dcc31367202da22953a6aa8818c30cc09c1c0cb..2d8a0559ebd3e351f68bd19440e56303bb8cad67 100644 --- a/cv/detection/oc_sort/paddlepaddle/README.md +++ b/cv/detection/oc_sort/paddlepaddle/README.md @@ -152,9 +152,13 @@ ln -s ../ByteTrack/datasets/mot/annotations ./ ### Install Dependencies ```bash -yum install mesa-libGL -y +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx -git clone https://github.com/PaddlePaddle/PaddleDetection.git +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ pip3 install -r requirements.txt diff --git a/cv/detection/picodet/paddlepaddle/README.md b/cv/detection/picodet/paddlepaddle/README.md index 5bcdedcb2932620c37a2631c01b11625c0d5f705..cee3cff1fd3b17decd168c8eba95ee2f390c227a 100644 --- a/cv/detection/picodet/paddlepaddle/README.md +++ b/cv/detection/picodet/paddlepaddle/README.md @@ -13,7 +13,13 @@ on resource-constrained devices. ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset diff --git a/cv/detection/pp-yoloe/paddlepaddle/README.md b/cv/detection/pp-yoloe/paddlepaddle/README.md index 817b1667c405f93fb4b07b01b6717abf8359f56d..eed132eaab66f211e97f7e7f69b056f90b472a40 100644 --- a/cv/detection/pp-yoloe/paddlepaddle/README.md +++ b/cv/detection/pp-yoloe/paddlepaddle/README.md @@ -14,11 +14,19 @@ for object detection tasks. ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset python3 dataset/coco/download_coco.py +or +ln -s /path/to/coco2017 dataset/coco ``` ### Install Dependencies diff --git a/cv/detection/pp_yoloe+/paddlepaddle/README.md b/cv/detection/pp_yoloe+/paddlepaddle/README.md index 851a0524ef8730e559377927ee1775f06bf5132f..e724119bc91e3642147d5280597ae2b1f388e1ad 100644 --- a/cv/detection/pp_yoloe+/paddlepaddle/README.md +++ b/cv/detection/pp_yoloe+/paddlepaddle/README.md @@ -13,10 +13,18 @@ real-time applications, offering superior detection performance across various s ### Prepare Resources ```bash +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + git clone -b release/2.7 https://github.com/PaddlePaddle/PaddleYOLO.git cd PaddleYOLO/ python3 dataset/coco/download_coco.py +or +ln -s /path/to/coco2017 dataset/coco ``` ### Install Dependencies diff --git a/cv/detection/pvanet/pytorch/ci/prepare.sh b/cv/detection/pvanet/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..42ca27f1ea33db695f09f44f2d64de9508f47956 --- /dev/null +++ b/cv/detection/pvanet/pytorch/ci/prepare.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +pip3 install 'scipy' 'matplotlib' 'pycocotools' 'opencv-python' 'easydict' 'tqdm' + +### Multiple GPUs on one machine +# bash train_pvanet_dist.sh --data-path /mnt/deepspark/data/datasets/coco2017/ --dataset coco + +### On single GPU +timeout 1800 python3 train.py --data-path /mnt/deepspark/data/datasets/coco2017/ --dataset coco \ No newline at end of file diff --git a/cv/detection/pvanet/pytorch/coco_eval.py b/cv/detection/pvanet/pytorch/coco_eval.py index a6e5547c118bf290c5e1e603594076b797e1180f..8004f54b39e0f31e9efa864934df9412f9db5059 100755 --- a/cv/detection/pvanet/pytorch/coco_eval.py +++ b/cv/detection/pvanet/pytorch/coco_eval.py @@ -5,7 +5,7 @@ import numpy as np import copy import time import torch -import torch._six +# import torch._six # pytorch 2.0后无此api from pycocotools.cocoeval import COCOeval from pycocotools.coco import COCO diff --git a/cv/detection/reppoints_mmdet/pytorch/README.md b/cv/detection/reppoints_mmdet/pytorch/README.md index b8849f69a677a4f6f83d71841d44c9daa1ef4931..82370f5b20ad10ed58ad96aca1aa7570097ed5c7 100644 --- a/cv/detection/reppoints_mmdet/pytorch/README.md +++ b/cv/detection/reppoints_mmdet/pytorch/README.md @@ -77,6 +77,5 @@ bash tools/dist_train.sh configs/reppoints/reppoints-moment_r101-dconv-c3-c5_fpn |-----------|------------|----------| | RepPoints | BI-V100 x8 | MAP=43.2 | -## References - -- [Paper](https://arxiv.org/abs/1904.11490) +## Reference +[mmdetection](https://github.com/open-mmlab/mmdetection) diff --git a/cv/detection/retinanet/paddlepaddle/README.md b/cv/detection/retinanet/paddlepaddle/README.md index 11e4287ad78fa5b25b836dc4a7e956830d31b741..9a007f676f8be5b62ba8086310d2488015d10bb4 100644 --- a/cv/detection/retinanet/paddlepaddle/README.md +++ b/cv/detection/retinanet/paddlepaddle/README.md @@ -13,7 +13,13 @@ single-stage approaches, offering an excellent balance between performance and e ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset diff --git a/cv/detection/retinanet/pytorch/ci/prepare.sh b/cv/detection/retinanet/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..bd4de78ab27d52e957ba9a8b1e88615b25d968fb --- /dev/null +++ b/cv/detection/retinanet/pytorch/ci/prepare.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +pip3 install 'scipy' 'matplotlib' 'pycocotools' 'opencv-python' 'easydict' 'tqdm' + +mkdir -p /root/.cache/torch/hub/checkpoints/ +cp /mnt/deepspark/data/checkpoints/resnet50-19c8e357.pth /root/.cache/torch/hub/checkpoints/resnet50-19c8e357.pth + +timeout 1800 bash train_retinanet_r50_dist.sh --data-path /mnt/deepspark/data/datasets/coco2017/ --dataset coco \ No newline at end of file diff --git a/cv/detection/rt-detr/pytorch/ci/prepare.sh b/cv/detection/rt-detr/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..24a332195403d614d0460def03be568dfdfe325b --- /dev/null +++ b/cv/detection/rt-detr/pytorch/ci/prepare.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +pip3 install -r requirements.txt + +mkdir -p dataset +ln -s /mnt/deepspark/data/datasets/coco2017 dataset/coco + +# training on single-gpu +export CUDA_VISIBLE_DEVICES=0 +timeout 1800 python3 tools/train.py -c configs/rtdetr/rtdetr_r50vd_6x_coco.yml + +# train on multi-gpu +# export CUDA_VISIBLE_DEVICES=0,1,2,3 +# torchrun --nproc_per_node=4 tools/train.py -c configs/rtdetr/rtdetr_r50vd_6x_coco.yml + +# val on multi-gpu +# export CUDA_VISIBLE_DEVICES=0,1,2,3 +# torchrun --nproc_per_node=4 tools/train.py -c configs/rtdetr/rtdetr_r50vd_6x_coco.yml -r path/to/checkpoint --test-only \ No newline at end of file diff --git a/cv/detection/solov2/paddlepaddle/README.md b/cv/detection/solov2/paddlepaddle/README.md index 2cda7326e8b96100234809bdd3d605532a5ec23b..8213cdc6bc5ca3a2314281cd3a852f4ffb093d7a 100644 --- a/cv/detection/solov2/paddlepaddle/README.md +++ b/cv/detection/solov2/paddlepaddle/README.md @@ -13,7 +13,13 @@ tasks while maintaining real-time capabilities, making it suitable for various c ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset @@ -47,3 +53,7 @@ python3 tools/train.py -c configs/solov2/solov2_r50_fpn_1x_coco.yml --eval | Model | GPU | FPS | Train Epochs | mAP | |--------|------------|------|--------------|------| | SOLOv2 | BI-V100 x8 | 6.39 | 12 | 35.4 | + +## Reference + +- [PaddleDetection](https://github.com/PaddlePaddle/PaddleDetection) \ No newline at end of file diff --git a/cv/detection/ssd/paddlepaddle/README.md b/cv/detection/ssd/paddlepaddle/README.md index ce64580a5a1a794e50e04831b330707f4d1a8e16..306f9c2cce8c4b83184d2ce8d0e3fe9baeedcc10 100644 --- a/cv/detection/ssd/paddlepaddle/README.md +++ b/cv/detection/ssd/paddlepaddle/README.md @@ -12,11 +12,19 @@ objects at different resolutions, offering a good balance between speed and accu ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset python3 dataset/coco/download_coco.py +or +ln -s /path/to/coco2017 dataset/coco ``` ### Install Dependencies diff --git a/cv/detection/ssd/pytorch/base/prepare.py b/cv/detection/ssd/pytorch/base/prepare.py index b85fb7522796f12be613445473c03efc830a0ccf..4b181436e6909c33d5e51d874b48ee85d48e9abb 100644 --- a/cv/detection/ssd/pytorch/base/prepare.py +++ b/cv/detection/ssd/pytorch/base/prepare.py @@ -6,7 +6,7 @@ import os.path as ospath from argparse import ArgumentParser, REMAINDER from functools import partial, wraps from typing import NamedTuple -import platform +# import platform # python3.5已弃用 # ========================================================= @@ -268,12 +268,27 @@ def install_extensions(): return exec_shell_cmds(cmds) +def get_linux_distro(): + try: + with open('/etc/os-release') as f: + for line in f: + if line.startswith('NAME='): + # 提取 NAME 字段的值（例如 "Ubuntu" 或 "CentOS"） + name = line.split('=')[1].strip().strip('"') + if 'Ubuntu' in name: + return 'Ubuntu' + elif 'CentOS' in name: + return 'CentOS' + else: + return name # 返回其他发行版名称 + except FileNotFoundError: + return 'Unknown Linux distribution' def pipelines(): - plat = [x.lower() for x in platform.platform().split("-")] - if "centos" in plat: + plat = get_linux_distro().lower() + if "centos" == plat: res = [install_yum_packages] - elif "debian" in plat or "ubuntu" in plat: + elif "ubuntu" == plat: res = [install_apt_packages] else: raise Exception("Invalid Platform, only support Centos and Debian!") diff --git a/cv/detection/ssd/pytorch/ci/prepare.sh b/cv/detection/ssd/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..7a9d113e809a917077f2d551bd4b61374af4686e --- /dev/null +++ b/cv/detection/ssd/pytorch/ci/prepare.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +mkdir -p /home/data/perf/ssd +ln -s /mnt/deepspark/data/datasets/coco /home/data/perf/ssd/ +cp /mnt/deepspark/data/checkpoints/resnet34-333f7ec4.pth /home/data/perf/ssd/ + +cd base +source ../iluvatar/config/environment_variables.sh +python3 prepare.py --name iluvatar --data_dir /home/data/perf/ssd +timeout 1800 bash run_training.sh --name iluvatar --config V100x1x8 --data_dir /home/data/perf/ssd --backbone_path /home/data/perf/ssd/resnet34-333f7ec4.pth \ No newline at end of file diff --git a/cv/detection/ssd/tensorflow/README.md b/cv/detection/ssd/tensorflow/README.md index fe9ff61da37bbd3cc7c5ce7fcecf49f3c7354089..921a0966e6fd07ee8bbd165224b7d0e4488324f4 100644 --- a/cv/detection/ssd/tensorflow/README.md +++ b/cv/detection/ssd/tensorflow/README.md @@ -32,6 +32,13 @@ dataset/VOCROOT/ VOCROOT is your path of the Pascal VOC Dataset. ```bash +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +cd dataset mkdir tfrecords pip3 install tf_slim python3 convert_voc_sample_tfrecords.py --dataset_directory=./ --output_directory=tfrecords --train_splits VOC2012_sample --validation_splits VOC2012_sample diff --git a/cv/detection/ssd/tensorflow/ci/prepare.sh b/cv/detection/ssd/tensorflow/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..363671f6a761881d6949ed6c3a463a5229ad80d3 --- /dev/null +++ b/cv/detection/ssd/tensorflow/ci/prepare.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +pip3 install tf_slim +cd dataset +mkdir tfrecords +ln -s /mnt/deepspark/data/datasets/VOC2012_sample ./ +python3 convert_voc_sample_tfrecords.py --dataset_directory=./ --output_directory=tfrecords --train_splits VOC2012_sample --validation_splits VOC2012_sample +cd .. +ln -s /mnt/deepspark/data/checkpoints/ssd-vgg16 ./model +python3 train_ssd.py --batch_size 16 \ No newline at end of file diff --git a/cv/detection/yolov3/paddlepaddle/README.md b/cv/detection/yolov3/paddlepaddle/README.md index 5c70713a60e821cef3b19ca0474f1190062e6970..b7f6ceadc1deb3b7fe9d9e675bf52b6dcd5efaa9 100644 --- a/cv/detection/yolov3/paddlepaddle/README.md +++ b/cv/detection/yolov3/paddlepaddle/README.md @@ -13,11 +13,19 @@ practical detection tasks. ### Prepare Resources ```bash -git clone https://github.com/PaddlePaddle/PaddleDetection.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone --recursive https://github.com/PaddlePaddle/PaddleDetection.git -b release2.6 --depth=1 cd PaddleDetection/ # Get COCO Dataset python3 dataset/coco/download_coco.py +or +ln -s /path/to/coco2017 dataset/coco ``` ### Install Dependencies @@ -42,3 +50,7 @@ python3 -u -m paddle.distributed.launch --gpus 0,1,2,3,4,5,6,7 \ --use_vdl=true \ --eval ``` + +## Reference + +- [PaddleDetection](https://github.com/PaddlePaddle/PaddleDetection) \ No newline at end of file diff --git a/cv/detection/yolov3/tensorflow/README.md b/cv/detection/yolov3/tensorflow/README.md index 92d9619c354152cf111ba75fa8626796c560a1a5..ccfbf08363cbb86a0a0fea0d3cc9f0e1b153c98a 100644 --- a/cv/detection/yolov3/tensorflow/README.md +++ b/cv/detection/yolov3/tensorflow/README.md @@ -40,6 +40,12 @@ Exporting loaded COCO weights as TF checkpoint(yolov3_coco.ckpt)[BaiduCloud](htt ### Install Dependencies ```bash +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + bash init_tf.sh ``` diff --git a/cv/detection/yolov3/tensorflow/ci/prepare.sh b/cv/detection/yolov3/tensorflow/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..f5329ce79930b44f37a22f193a6b942acffede88 --- /dev/null +++ b/cv/detection/yolov3/tensorflow/ci/prepare.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +bash init_tf.sh +ln -s /mnt/deepspark/data/datasets/VOC ./ +bash ./run_training.sh \ No newline at end of file diff --git a/cv/detection/yolov3/tensorflow/train_fast.py b/cv/detection/yolov3/tensorflow/train_fast.py index 5992fd4ae987c769e6d5bb21acca0b0ddf1f5ef2..9217bd05e76a5282168636c403cf315d92aae2fd 100644 --- a/cv/detection/yolov3/tensorflow/train_fast.py +++ b/cv/detection/yolov3/tensorflow/train_fast.py @@ -227,6 +227,8 @@ class YoloTrain(object): self.summary_writer.add_summary(summary, global_step_val) pbar.set_description("train loss: %.2f" % train_step_loss) + # fix epoch not assign + epoch = 1+self.first_stage_epochs+self.second_stage_epochs train_epoch_loss = np.mean(train_epoch_loss) log_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) print("=> Epoch: %2d Time: %s Train loss: %.2f..." diff --git a/cv/detection/yolov5/paddlepaddle/README.md b/cv/detection/yolov5/paddlepaddle/README.md index 4e2b338ffc23b77ab445da2348d6e574008c9f62..dc7b4891528a3eb9468f93754acede200d9c89f6 100644 --- a/cv/detection/yolov5/paddlepaddle/README.md +++ b/cv/detection/yolov5/paddlepaddle/README.md @@ -12,11 +12,18 @@ and efficient inference, making it popular for real-time detection tasks across ### Prepare Resources ```bash -cd deepsparkhub/cv/detection/yolov5/paddlepaddle/ -git clone -b release/2.5 https://github.com/PaddlePaddle/PaddleYOLO.git +# Install libGL +## CentOS +yum install -y mesa-libGL +## Ubuntu +apt install -y libgl1-mesa-glx + +git clone -b release/2.7 --depth=1 https://github.com/PaddlePaddle/PaddleYOLO.git cd PaddleYOLO/ python3 dataset/coco/download_coco.py +or +ln -s /path/to/coco2017 dataset/coco ``` ### Install Dependencies diff --git a/cv/detection/yolov5/pytorch/ci/prepare.sh b/cv/detection/yolov5/pytorch/ci/prepare.sh new file mode 100644 index 0000000000000000000000000000000000000000..bc538008f0f51e437a4bac47ec80ce5f4193ebb0 --- /dev/null +++ b/cv/detection/yolov5/pytorch/ci/prepare.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +## install libGL +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +if [[ ${ID} == "ubuntu" ]]; then + apt install -y libgl1-mesa-glx +elif [[ ${ID} == "centos" ]]; then + yum install -y mesa-libGL +else + echo "Not Support Os" +fi + +## clone yolov5 and install +git clone https://gitee.com/deep-spark/deepsparkhub-GPL.git +cd deepsparkhub-GPL/cv/detection/yolov5/pytorch/ +pip3 install -r requirements.txt +pip3 install matplotlib==3.8.2 +pip3 install numpy==1.26.4 +pip3 install pandas==2.0.0 +wandb disabled +pip3 install pycocotools + +mkdir -p datasets +ln -s /mnt/deepspark/data/datasets/coco ./datasets/coco + +### On single GPU +timeout 1800 python3 train.py --data ./data/coco.yaml --batch-size 32 --cfg ./models/yolov5s.yaml --weights '' + +### On single GPU (AMP) +# python3 train.py --data ./data/coco.yaml --batch-size 32 --cfg ./models/yolov5s.yaml --weights '' --amp + +### Multiple GPUs on one machine +# python3 -m torch.distributed.launch --nproc_per_node 8 \ +# train.py \ +# --data ./data/coco.yaml \ +# --batch-size 64 \ +# --cfg ./models/yolov5s.yaml --weights '' \ +# --device 0,1,2,3,4,5,6,7 + +# YOLOv5m +# bash run.sh + +# ### Multiple GPUs on one machine (AMP) +# python3 -m torch.distributed.launch --nproc_per_node 8 \ +# train.py \ +# --data ./data/coco.yaml \ +# --batch-size 256 \ +# --cfg ./models/yolov5s.yaml --weights '' \ +# --device 0,1,2,3,4,5,6,7 --amp + + +## Test the detector +# python3 detect.py --source ./data/images/bus.jpg --weights yolov5s.pt --img 640 +# python3 detect.py --source ./data/images/zidane.jpg --weights yolov5s.pt --img 640 \ No newline at end of file diff --git a/tests/run_test.py b/tests/run_test.py new file mode 100644 index 0000000000000000000000000000000000000000..cd0146d6089a84abe85d53024410708496097cf5 --- /dev/null +++ b/tests/run_test.py @@ -0,0 +1,300 @@ +# Copyright (c) 2024, Shanghai Iluvatar CoreX Semiconductor Co., Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml +import subprocess +import json +import re +import time +import logging +import os +import sys +import argparse + +def is_debug(): + is_debug_flag = os.environ.get("IS_DEBUG") + if is_debug_flag and is_debug_flag.lower()=="true": + return True + else: + return False + +def main(): + parser = argparse.ArgumentParser(description="") + parser.add_argument("--model", type=str, help="model name, e.g: alexnet") + args = parser.parse_args() + + if args.model: + test_model = args.model + else: + test_model = os.environ.get("TEST_CASE") + test_framework = os.environ.get("FRAMEWORK") + test_category = os.environ.get("category") + logging.info(f"Test case to run: {test_model}") + if not test_model: + logging.error("test model case is empty") + sys.exit(-1) + + model = get_model_config(test_model, test_framework, test_category) + if not model: + logging.error("mode config is empty") + sys.exit(-1) + + result = {} + if model["category"] == "cv/classification" and model["framework"] == "pytorch": + logging.info(f"Start running {model['model_name']} test case:\n{json.dumps(model, indent=4)}") + result = run_clf_testcase(model) + check_model_result(result) + logging.debug(f"The result of {model['model_name']} is\n{json.dumps(result, indent=4)}") + logging.info(f"End running {model['model_name']} test case.") + + if model["category"] == "cv/detection" and (model["framework"] == "pytorch" or model["framework"] == "paddlepaddle"): + logging.info(f"Start running {model['model_name']} test case:\n{json.dumps(model, indent=4)}") + result = run_detec_testcase(model) + check_model_result(result) + logging.debug(f"The result of {model['model_name']} is\n{json.dumps(result, indent=4)}") + logging.info(f"End running {model['model_name']} test case.") + + logging.info(f"Full text result: {result}") + +def get_model_config(model_name, framework, category): + print(f"model_name: {model_name}, framework: {framework}, category: {category}") + with open('all_deepsparkhub_model_info.json', mode='r', encoding='utf-8') as config_file: + config_data = json.load(config_file) + + for model in config_data["models"]: + # TODO: 模型名称+模型框架+模型类别才能唯一确定一个模型 + if model["model_name"] == model_name.lower() and model["framework"] == framework.lower() and model["category"] == category.lower(): + return model + return + +def check_model_result(result): + status = "PASS" + for prec in ["fp16", "int8"]: + if prec in result["result"]: + if result["result"][prec]["status"] == "FAIL": + status = "FAIL" + break + result["status"] = status + +def run_detec_testcase(model): + model_name = model["model_name"] + result = { + "name": model_name, + "framework": model["framework"], + "toolbox": model["toolbox"], + "category": "cv/detection", + "result": {}, + } + is_mmdetection = True if model["toolbox"] == "mmdetection" else False + is_yolov = True if model["toolbox"] == "yolov" else False + is_paddledetection = True if model["toolbox"] == "PaddleDetection" else False + deepsparkhub_path = model["deepsparkhub_path"].replace("deepsparkhub/", "") + if is_mmdetection: + # 选择使用atss作为个例 + prepare_script = f""" + apt install -y libgl1-mesa-glx + cd ../cv/detection/atss_mmdet/pytorch/ + cp -r /mnt/deepspark/data/3rd_party/mmdetection-v3.3.0 ./mmdetection > /dev/null 2>&1 + mkdir -p /root/.cache/torch/hub/checkpoints/ + cp /mnt/deepspark/data/checkpoints/resnet50-0676ba61.pth /root/.cache/torch/hub/checkpoints/resnet50-0676ba61.pth + mkdir -p mmdetection/data + ln -s /mnt/deepspark/data/datasets/coco2017 mmdetection/data/coco + cd mmdetection + pip install -v -e . + sed -i 's/python /python3 /g' tools/dist_train.sh + timeout 1800 bash tools/dist_train.sh configs/atss/atss_r50_fpn_1x_coco.py 4 + """ + elif is_paddledetection: + # 选择使用CenterNet作为个例 + prepare_script = f""" + apt install -y libgl1-mesa-glx + cd ../cv/detection/centernet/paddlepaddle/ + cp -r /mnt/deepspark/data/3rd_party/PaddleDetection-release-2.6 ./PaddleDetection + cd PaddleDetection + pip install -r requirements.txt + python3 setup.py install + mv dataset/coco dataset/coco-bak + ln -s /mnt/deepspark/data/datasets/coco2017 dataset/coco + timeout 1800 python3 tools/train.py -c configs/centernet/centernet_r50_140e_coco.yml --eval + """ + elif is_yolov: + # 选择使用yolov5作为个例 + prepare_script = f""" + cd ../cv/detection/yolov5/pytorch/ + bash ci/prepare.sh + """ + else: + prepare_script = f""" + cd ../{deepsparkhub_path} + bash ci/prepare.sh + """ + + # add pip list info when in debug mode + if is_debug(): + pip_list_script = "pip list | grep -E 'numpy|transformer|igie|mmcv|onnx'\n" + prepare_script = pip_list_script + prepare_script + pip_list_script + + logging.info(f"Start running {model_name} test case") + r, t = run_script(prepare_script) + sout = r.stdout + prec = "fp16" + pattern = r"Average Precision \(AP\) @\[ (IoU=0.50[:\d.]*)\s*\| area= all \| maxDets=\s?\d+\s?\] =\s*([\d.]+)" + matches = re.findall(pattern, sout) + epoch_pattern = [r"train: \[", r"Epoch: \[", r".*Epoch\s+gpu_mem", r"total_loss: "] + combined_pattern = re.compile("|".join(epoch_pattern)) + epoch_matches = bool(combined_pattern.search(sout)) + + if matches: + # Get last match and convert to floats + last_map = map(float, matches[-1]) + print(f"MAP: {last_map}") + result["result"].setdefault(prec, {"status": "PASS"}) + result["result"][prec]["MAP"] = last_map + elif epoch_matches: + result["result"].setdefault(prec, {"status": "PASS"}) + result["result"][prec]["MAP"] = "train timeout" + else: + result["result"].setdefault(prec, {"status": "FAIL"}) + print("No match found.") + + result["result"][prec]["Cost time (s)"] = t + logging.debug(f"matchs:\n{matches}\nepoch_matches:\n{epoch_matches}") + return result + +def run_clf_testcase(model): + model_name = model["model_name"] + result = { + "name": model_name, + "framework": model["framework"], + "toolbox": model["toolbox"], + "category": "cv/classification", + "result": {}, + } + is_torchvision = True if model["toolbox"].lower() == "torchvision" else False + is_mmpretrain = True if model["toolbox"].lower() == "mmpretrain" else False + is_paddleclas = True if model["toolbox"].lower() == "paddleclas" else False + is_tf_benchmarks = True if model["toolbox"].lower() == "tensorflow/benchmarks" else False + deepsparkhub_path = model["deepsparkhub_path"].replace("deepsparkhub/", "") + dataset_path = "/mnt/deepspark/data/datasets/imagenet" + + # # add pip list info when in debug mode + # if is_debug(): + # pip_list_script = "pip list | grep -E 'numpy|transformer|igie|mmcv|onnx'\n" + # prepare_script = pip_list_script + prepare_script + pip_list_script + + logging.info(f"Start running {model_name} test case") + if is_torchvision: + # 选择使用googlenet作为个例 + prepare_script = f""" + cd ../{deepsparkhub_path} + timeout 1800 python3 -m torch.distributed.launch --nproc_per_node=4 --use_env train.py --data-path {dataset_path} --model googlenet --batch-size 512 + """ + elif is_mmpretrain: + # 选择使用mocov2作为个例 + prepare_script = f""" + apt install -y libgl1-mesa-glx + cd ../cv/classification/mocov2/pytorch/ + cp -r /mnt/deepspark/data/3rd_party/mmpretrain-v1.2.0 ./mmpretrain > /dev/null 2>&1 + cd mmpretrain + python3 setup.py install + mkdir -p data + ln -s /mnt/deepspark/data/datasets/imagenet data/ + ln -s /mnt/deepspark/data/checkpoints/mocov2_resnet50_8xb32-coslr-200e_in1k_20220825-b6d23c86.pth ./ + sed -i 's@model = dict(backbone=dict(frozen_stages=4))@model = dict(backbone=dict(frozen_stages=4,init_cfg=dict(type='\''Pretrained'\'', checkpoint='\''./mocov2_resnet50_8xb32-coslr-200e_in1k_20220825-b6d23c86.pth'\'', prefix='\''backbone.'\'')))@' configs/mocov2/benchmarks/resnet50_8xb32-linear-steplr-100e_in1k.py + timeout 1800 python3 tools/train.py configs/mocov2/mocov2_resnet50_8xb32-coslr-200e_in1k.py + """ + elif is_paddleclas: + # 选择使用googlenet作为个例 + prepare_script = f""" + apt install -y libgl1-mesa-glx + cd ../cv/classification/googlenet/paddlepaddle/ + cp -r /mnt/deepspark/data/3rd_party/PaddleClas-release-2.6 ./PaddleClas > /dev/null 2>&1 + cd PaddleClas + pip3 install -r requirements.txt + python3 setup.py install + mkdir -p dataset + ln -s /mnt/deepspark/data/datasets/imagenet dataset/ILSVRC2012 + timeout 1800 python3 -u -m paddle.distributed.launch --gpus=0,1,2,3 tools/train.py -c ppcls/configs/ImageNet/Inception/GoogLeNet.yaml -o Arch.pretrained=False -o Global.device=gpu + """ + elif is_tf_benchmarks: + # 选择使用alexnet作为个例 + prepare_script = f""" + apt install -y libgl1-mesa-glx + cd ../cv/classification/alexnet/tensorflow/ + ln -s /mnt/deepspark/data/datasets/imagenet_tfrecord/ILSVRC2012 dataset/imagenet_tfrecord + timeout 1800 bash run_train_alexnet_imagenet.sh + """ + else: + prepare_script = f""" + cd ../{deepsparkhub_path} + ln -s /mnt/deepspark/data/datasets/imagenet ./ + bash ci/prepare.sh + """ + r, t = run_script(prepare_script) + sout = r.stdout + prec = "fp16" + pattern = re.compile(r'\* Acc@1 (\d+\.\d+) Acc@5 (\d+\.\d+)') + matches = pattern.findall(sout) + epoch_pattern = [r"Epoch: \[", r"Epoch\(train\) \[", r".*TrainEpoch-", r"INFO Train:.*", r".*\[Train\]\[Epoch.*", r".*images\/sec.*"] + combined_pattern = re.compile("|".join(epoch_pattern)) + epoch_matches = bool(combined_pattern.search(sout)) + + if matches: + # Get last match and convert to floats + last_acc1, last_acc5 = map(float, matches[-1]) + print(f"Acc@1: {last_acc1}, Acc@5: {last_acc5}") + result["result"].setdefault(prec, {"status": "PASS"}) + result["result"][prec]["acc1"] = last_acc1 + result["result"][prec]["acc5"] = last_acc5 + elif epoch_matches: + result["result"].setdefault(prec, {"status": "PASS"}) + result["result"][prec]["acc1"] = "train timeout" + result["result"][prec]["acc5"] = "train timeout" + else: + result["result"].setdefault(prec, {"status": "FAIL"}) + print("No match found.") + + result["result"][prec]["Cost time (s)"] = t + logging.debug(f"matchs:\n{matches}") + return result + +def get_metric_result(str): + if str: + return json.loads(str.replace("'", "\""))["metricResult"] + return None + +def run_script(script): + start_time = time.perf_counter() + result = subprocess.run( + script, shell=True, capture_output=True, text=True, executable="/bin/bash" + ) + end_time = time.perf_counter() + execution_time = end_time - start_time + logging.debug(f"执行命令：\n{script}") + logging.debug("执行时间: {:.4f} 秒".format(execution_time)) + logging.debug(f"标准输出: {result.stdout}") + logging.debug(f"标准错误: {result.stderr}") + logging.debug(f"返回码: {result.returncode}") + return result, execution_time + +if __name__ == "__main__": + # 配置日志 + debug_level = logging.DEBUG if is_debug() else logging.INFO + logging.basicConfig( + handlers=[logging.FileHandler("output.log"), logging.StreamHandler()], + level=debug_level, + format="%(asctime)s - %(levelname)s - %(message)s", + ) + main()