From 1fe6a91bf1e40cc637ea7901dee8baf5b0cf8716 Mon Sep 17 00:00:00 2001
From: liuyihang <1905527319@qq.com>
Date: Fri, 22 Jul 2022 01:17:03 +0000
Subject: [PATCH] =?UTF-8?q?add=20Resnet50v1.5=5FID1721=5Ffor=5FTensorFlow/?=
 =?UTF-8?q?test/train=5Fand=5Feval=5F8p.sh.=20=E7=BD=91=E7=BB=9C=E6=9C=89d?=
 =?UTF-8?q?ata=5Fformat=E7=9A=84=E5=88=A4=E6=96=AD=EF=BC=8C=E6=88=91?=
 =?UTF-8?q?=E4=BB=AC=E7=9A=84=E6=AC=A7=E6=8B=89=E7=B3=BB=E7=BB=9F=E8=A3=85?=
 =?UTF-8?q?=E7=9A=84tensorflow=E6=98=AF=E9=9D=9E=E5=BC=80=E6=BA=90?=
 =?UTF-8?q?=E7=9A=84=EF=BC=8C=E9=BB=98=E8=AE=A4=E6=98=AF=E4=B8=8D=E6=94=AF?=
 =?UTF-8?q?=E6=8C=81cuda=E7=9A=84=EF=BC=8C=E4=BD=86=E6=98=AFubuntu?=
 =?UTF-8?q?=E7=9A=84=E7=B3=BB=E7=BB=9F=E8=A3=85=E7=9A=84=E6=98=AF=E5=BC=80?=
 =?UTF-8?q?=E6=BA=90=E7=9A=84=EF=BC=8C=E6=94=AF=E6=8C=81cuda=EF=BC=8C?=
 =?UTF-8?q?=E5=AF=BC=E8=87=B4=E6=9C=89=E4=BA=9B=E7=BD=91=E7=BB=9C=E5=8F=AF?=
 =?UTF-8?q?=E4=BB=A5=E5=9C=A8=E6=AC=A7=E6=8B=89=E7=B3=BB=E7=BB=9F=E8=B7=91?=
 =?UTF-8?q?=E9=80=9A=EF=BC=8C=E5=9C=A8ubuntu=E4=B8=8A=E9=9D=A2=E4=BC=9A?=
 =?UTF-8?q?=E6=8A=A5=E9=94=99=E3=80=82=E7=8E=B0=E5=9C=A8=E5=A2=9E=E5=8A=A0?=
 =?UTF-8?q?=E4=B8=80=E4=B8=AA=E8=84=9A=E6=9C=AC=EF=BC=8C=E5=B0=86data=5Ffo?=
 =?UTF-8?q?rmat=E5=86=99=E6=AD=BB=E4=BC=A0=E5=85=A5=E3=80=82=E9=AA=8C?=
 =?UTF-8?q?=E8=AF=81=E5=8A=9F=E8=83=BD=E3=80=82=E4=B8=AA=E4=BA=BA=E4=BB=93?=
 =?UTF-8?q?=E5=B7=B2=E9=AA=8C=E8=AF=81=EF=BC=8C=E5=8A=9F=E8=83=BDok?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../test/train_and_eval_8p.sh                 | 178 ++++++++++++++++++
 1 file changed, 178 insertions(+)
 create mode 100644 TensorFlow/built-in/cv/image_classification/Resnet50v1.5_ID1721_for_TensorFlow/test/train_and_eval_8p.sh

diff --git a/TensorFlow/built-in/cv/image_classification/Resnet50v1.5_ID1721_for_TensorFlow/test/train_and_eval_8p.sh b/TensorFlow/built-in/cv/image_classification/Resnet50v1.5_ID1721_for_TensorFlow/test/train_and_eval_8p.sh
new file mode 100644
index 000000000..73206a163
--- /dev/null
+++ b/TensorFlow/built-in/cv/image_classification/Resnet50v1.5_ID1721_for_TensorFlow/test/train_and_eval_8p.sh
@@ -0,0 +1,178 @@
+#!/bin/bash
+#当前路径,不需要修改
+cur_path=`pwd`
+#集合通信参数,不需要修改
+
+#export ASCEND_SLOG_PRINT_TO_STDOUT=1
+export RANK_SIZE=8
+export JOB_ID=10087
+export RANK_TABLE_FILE=$cur_path/../official/r1/resnet/scripts/8p.json
+RANK_ID_START=0
+
+
+# 数据集路径,保持为空,不需要修改
+data_path=""
+
+#设置默认日志级别,不需要修改
+#export ASCEND_GLOBAL_LOG_LEVEL_ETP_ETP=3
+
+#基础参数，需要模型审视修改
+#网络名称，同目录名称
+Network="Resnet50v1.5_ID1721_for_TensorFlow"
+#训练epochs
+train_epochs=90
+#训练steps
+train_steps=1250
+#训练batch_size
+batch_size=128
+
+#维测参数，precision_mode需要模型审视修改
+#precision_mode="allow_mix_precision"
+#维持参数，以下不需要修改
+over_dump=False
+data_dump_flag=False
+data_dump_step="10"
+profiling=False
+autotune=False
+
+# 帮助信息，不需要修改
+if [[ $1 == --help || $1 == -h ]];then
+    echo"usage:./train_performance_1p.sh <args>"
+    echo " "
+    echo "parameter explain:
+    --precision_mode         precision mode(allow_fp32_to_fp16/force_fp16/must_keep_origin_dtype/allow_mix_precision)
+    -h/--help		         show help message
+    "
+    exit 1
+fi
+
+#参数校验，不需要修改
+for para in $*
+do
+    if [[ $para == --precision_mode* ]];then
+        precision_mode=`echo ${para#*=}`
+    elif [[ $para == --over_dump* ]];then
+        over_dump=`echo ${para#*=}`
+        over_dump_path=${cur_path}/output/overflow_dump
+        mkdir -p ${over_dump_path}
+    elif [[ $para == --data_dump_flag* ]];then
+        data_dump_flag=`echo ${para#*=}`
+        data_dump_path=${cur_path}/output/data_dump
+        mkdir -p ${data_dump_path}
+    elif [[ $para == --data_dump_step* ]];then
+        data_dump_step=`echo ${para#*=}`
+    elif [[ $para == --profiling* ]];then
+        profiling=`echo ${para#*=}`
+        profiling_dump_path=${cur_path}/output/profiling
+        mkdir -p ${profiling_dump_path}
+    elif [[ $para == --autotune* ]];then
+        autotune=`echo ${para#*=}`
+        mv $install_path/fwkacllib/data/rl/Ascend910/custom $install_path/fwkacllib/data/rl/Ascend910/custom_bak
+        mv $install_path/fwkacllib/data/tiling/Ascend910/custom $install_path/fwkacllib/data/tiling/Ascend910/custom_bak
+        autotune_dump_path=${cur_path}/output/autotune_dump
+        mkdir -p ${autotune_dump_path}/GA
+        mkdir -p ${autotune_dump_path}/rl
+        cp -rf $install_path/fwkacllib/data/tiling/Ascend910/custom ${autotune_dump_path}/GA/
+        cp -rf $install_path/fwkacllib/data/rl/Ascend910/custom ${autotune_dump_path}/RL/
+    elif [[ $para == --data_path* ]];then
+        data_path=`echo ${para#*=}`
+    fi
+done
+
+
+#data_path='../'
+#校验是否传入data_path,不需要修改
+if [[ $data_path == "" ]];then
+    echo "[Error] para \"data_path\" must be confing"
+    exit 1
+fi
+
+cd $cur_path/../official/r1/resnet
+
+#训练开始时间，不需要修改
+start_time=$(date +%s)
+
+#进入训练脚本目录，需要模型审视修改
+for((RANK_ID=$RANK_ID_START;RANK_ID<$((RANK_SIZE+RANK_ID_START));RANK_ID++));
+do
+    #设置环境变量，不需要修改
+    echo "Device ID: $RANK_ID"
+    export RANK_ID=$RANK_ID
+    export ASCEND_DEVICE_ID=$RANK_ID
+    ASCEND_DEVICE_ID=$RANK_ID
+
+    export DEVICE_ID=$RANK_ID
+	DEVICE_INDEX=$RANK_ID
+    export DEVICE_INDEX=${DEVICE_INDEX}
+    
+    #创建DeviceID输出目录，不需要修改
+    if [ -d ${cur_path}/output/${ASCEND_DEVICE_ID} ];then
+        rm -rf ${cur_path}/output/${ASCEND_DEVICE_ID}
+        mkdir -p ${cur_path}/output/$ASCEND_DEVICE_ID/ckpt
+    else
+        mkdir -p ${cur_path}/output/$ASCEND_DEVICE_ID/ckpt
+    fi
+
+	#执行训练脚本，以下传参不需要修改，其他需要模型审视修改
+    #--data_dir, --model_dir, --precision_mode, --over_dump, --over_dump_path，--data_dump_flag，--data_dump_step，--data_dump_path，--profiling，--profiling_dump_path，--autotune
+    
+    nohup python3 imagenet_main.py \
+        --resnet_size=50 \
+        --resnet_version=1 \
+        --max_train_steps=$train_steps \
+        --hooks=ExamplesPerSecondHook \
+        --data_dir=$data_path \
+        --train_epochs=$train_epochs \
+		--batch_size=$batch_size \
+        --epochs_between_evals 1 \
+        --data_dir=$data_path \
+        --model_dir=${cur_path}/output/${ASCEND_DEVICE_ID}/   > ${cur_path}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log 2>&1 &
+done 
+wait
+
+#训练结束时间，不需要修改
+end_time=$(date +%s)
+e2e_time=$(( $end_time - $start_time ))
+
+
+#结果打印，不需要修改
+echo "------------------ Final result ------------------"
+#输出性能FPS，需要模型审视修改
+FPS=`grep "tensorflow:steps" $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|awk -F 'FPS:' '{print $2}'|cut -c 1-7|awk 'NR>2'|awk '{sum+=$1} END {print  sum/NR}'`
+#打印，不需要修改
+echo "Final Performance images/sec : $FPS"
+echo "E2E Training Duration sec : $e2e_time"
+
+#精度计算，需要根据网络修改
+train_accuracy=`grep "name': 'accuracy'" $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log|awk '{print $6}'|tr -d ','|tail -1`
+
+#性能看护结果汇总
+#训练用例信息，不需要修改
+BatchSize=${batch_size}
+DeviceType=`uname -m`
+CaseName=${Network}_bs${BatchSize}_${RANK_SIZE}'p'_'acc'_train_and_eval
+
+#获取性能数据，不需要修改
+#吞吐量
+ActualFPS=${FPS}
+#单迭代训练时长
+TrainingTime=`awk 'BEGIN{printf "%.2f\n",'${batch_size}'*1000/'${ActualFPS}'}'`
+
+
+#从train_$ASCEND_DEVICE_ID.log提取Loss到train_${CaseName}_loss.txt中，需要根据模型审视
+grep 'loss =' $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|awk -F 'loss =' '{print $2}'|tr -d ',' |awk '{print $1}' > $cur_path/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt
+#最后一个迭代loss值，不需要修改
+ActualLoss=`awk 'END {print}' $cur_path/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt`
+
+
+#关键信息打印到${CaseName}.log中，不需要修改
+echo "Network = ${Network}" > $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
+echo "RankSize = ${RANK_SIZE}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
+echo "BatchSize = ${BatchSize}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
+echo "DeviceType = ${DeviceType}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
+echo "CaseName = ${CaseName}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
+echo "ActualFPS = ${ActualFPS}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
+echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
+echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
+echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
+echo "trainAccuracy = ${train_accuracy}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
-- 
Gitee