diff --git a/TensorFlow/built-in/cv/image_classification/MiniGo_ID0629_for_TensorFlow/README.md b/TensorFlow/built-in/cv/image_classification/MiniGo_ID0629_for_TensorFlow/README.md index 44ac5abee4decf4405834e142b293ff88f17a1ef..9964bcbfe0fbf4665d60294d83a6b88a0994da30 100644 --- a/TensorFlow/built-in/cv/image_classification/MiniGo_ID0629_for_TensorFlow/README.md +++ b/TensorFlow/built-in/cv/image_classification/MiniGo_ID0629_for_TensorFlow/README.md @@ -195,6 +195,10 @@ pip3 install requirements.txt 4. 8卡训练 4.1 首先检查 MiniGo_ID0629_for_TensorFlow/test 目录下是否有存在8卡IP的json配置文件 "8p.json" + (export CMxxx环境变量方式拉起训练无需检查) + + 环境变量方式拉起详情请参考: + https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/63RC1alpha002/tfmoddevg/tfmigr1/atlasmprtg_13_9036.html ​ 4.2 设置单卡训练参数(脚本位于./MiniGo_ID0629_for_TensorFlow/test/train_full_8p.sh),示例如下。 @@ -207,7 +211,9 @@ pip3 install requirements.txt ​ 4.3 单卡训练指令(脚本位于./MiniGo_ID0629_for_TensorFlow/test/train_full_8p.sh) ``` - bash train_full_8p.sh --data_path=xx + bash train_full_8p.sh --data_path=xx(ranktable方式) + bash train_full_8p.sh --data_path=xx --one_node_ip=本机IP(去ranktable--CM环境变量方式) + 数据集路径默认为 MiniGo_ID0629_for_TensorFlow/outputs/data/selfplay(即Step2自对弈的生成路径,不建议改动) 数据集应有如下结构(数据切分可能不同),配置data_path时需指定为selfplay这一层,例:--data_path=./outputs/data/selfplay ├─selfplay diff --git a/TensorFlow/built-in/cv/image_classification/MiniGo_ID0629_for_TensorFlow/test/train_full_8p.sh b/TensorFlow/built-in/cv/image_classification/MiniGo_ID0629_for_TensorFlow/test/train_full_8p.sh index 8cd78e2f2b804957f26e967acf45a7359b05c13e..37476144b7e243ca19afef906f1edc40b2478316 100644 --- a/TensorFlow/built-in/cv/image_classification/MiniGo_ID0629_for_TensorFlow/test/train_full_8p.sh +++ b/TensorFlow/built-in/cv/image_classification/MiniGo_ID0629_for_TensorFlow/test/train_full_8p.sh @@ -43,7 +43,9 @@ do name_bind="_bindcore" elif [[ $para == --dynamic_input* ]];then dynamic_input=`echo ${para#*=}` - fi + elif [[ $para == --one_node_ip* ]];then + one_node_ip=`echo ${para#*=}` + fi done if [[ $data_path == "" ]];then @@ -59,17 +61,25 @@ python3 bootstrap.py --work_dir=$cur_path/estimator_working_dir --export_path=$c wait export ASCEND_DEVICE_ID=0 -export RANK_SIZE=8 -export RANK_TABLE_FILE="${cur_path}/test/8p.json" +export RANK_SIZES=8 +#export RANK_TABLE_FILE="${cur_path}/test/8p.json" export JOB_ID=10086 +#8p训练必须参数(本机IP) +one_node_ip=$one_node_ip +#新增适配集群环境变量 +export CM_CHIEF_IP=${one_node_ip} #主节点ip,所有服务器一致 +export CM_CHIEF_PORT=29688 #通信端口,所有服务器一致 +export CM_CHIEF_DEVICE=0 #配置为0,配置主卡,类似于主节点,所有服务器一致 +export CM_WORKER_SIZE=8 #卡数,单机为8,所有服务器一致 +export CM_WORKER_IP=${one_node_ip} #当前服务器ip,不同环境ip不同 start=$(date +%s) # 8P训练模式 for i in 0 1 2 3 4 5 6 7 do #设置环境变量 - export RANK_ID=$i + export RANK_IDS=$i export ASCEND_DEVICE_ID=$i ASCEND_DEVICE_ID=$i echo "Device ID: $ASCEND_DEVICE_ID" @@ -83,8 +93,8 @@ do echo $ASCEND_DEVICE_ID #(Step3)训练 corenum=`cat /proc/cpuinfo |grep 'processor' |wc -l` - let a=RANK_ID*${corenum}/8 - let b=RANK_ID+1 + let a=RANK_IDS*${corenum}/8 + let b=RANK_IDS+1 let c=b*${corenum}/8-1 if [ "x${bind_core}" != x ];then bind_core="taskset -c $a-$c" diff --git a/TensorFlow/built-in/cv/image_classification/ResNet50_ID0058_for_TensorFlow/test/train_full_8p.sh b/TensorFlow/built-in/cv/image_classification/ResNet50_ID0058_for_TensorFlow/test/train_full_8p.sh index 254cad2ad44821779738f3a202600f7f8c65f3ee..d00a27957f442fd9afdb42266506ba11a3d71111 100644 --- a/TensorFlow/built-in/cv/image_classification/ResNet50_ID0058_for_TensorFlow/test/train_full_8p.sh +++ b/TensorFlow/built-in/cv/image_classification/ResNet50_ID0058_for_TensorFlow/test/train_full_8p.sh @@ -5,8 +5,8 @@ cur_path=`pwd` #集合通信参数,不需要修改 #保证rank table file 文件rank_table_8p.json存放在和test同级的configs目录下 -export RANK_SIZE=8 -export RANK_TABLE_FILE=${cur_path}/../configs/8p.json +export RANK_SIZES=8 +#export RANK_TABLE_FILE=${cur_path}/../configs/8p.json export JOB_ID=10087 RANK_ID_START=0 @@ -21,7 +21,7 @@ Network="ResNet50_ID0058_for_TensorFlow" export HCCL_CONNECT_TIMEOUT=600 corenum=`cat /proc/cpuinfo |grep "processor"|wc -l` export RANK_INDEX=0 -export RANK_ID=0 +export RANK_IDS=0 config_file=res50_256bs_8p_eval iterations_per_loop=100 @@ -86,9 +86,20 @@ do elif [[ $para == --bind_core* ]]; then bind_core=`echo ${para#*=}` name_bind="_bindcore" + elif [[ $para == --one_node_ip* ]];then + one_node_ip=`echo ${para#*=}` fi done +#8p训练必须参数(本机IP) +one_node_ip=$one_node_ip +#新增适配集群环境变量 +export CM_CHIEF_IP=${one_node_ip} #主节点ip,所有服务器一致 +export CM_CHIEF_PORT=29688 #通信端口,所有服务器一致 +export CM_CHIEF_DEVICE=0 #配置为0,配置主卡,类似于主节点,所有服务器一致 +export CM_WORKER_SIZE=8 #卡数,单机为8,所有服务器一致 +export CM_WORKER_IP=${one_node_ip} #当前服务器ip,不同环境ip不同 + #校验是否传入data_path,不需要修改 if [[ $data_path == "" ]];then echo "[Error] para \"data_path\" must be confing" @@ -104,16 +115,16 @@ fi #训练开始时间,不需要修改 start_time=$(date +%s) - +sed -i 's/RANK_SIZE/RANK_SIZES/g' ../src/data_loader/resnet50/data_loader.py #进入训练脚本目录,需要模型审视修改 cd $cur_path/.. -for((RANK_ID=$RANK_ID_START;RANK_ID<$((RANK_SIZE+RANK_ID_START));RANK_ID++)); +for((RANK_IDS=$RANK_ID_START;RANK_IDS<$((RANK_SIZES+RANK_ID_START));RANK_IDS++)); do #设置环境变量,不需要修改 - echo "Device ID: $RANK_ID" - # export RANK_ID=$RANK_ID - export ASCEND_DEVICE_ID=$RANK_ID - ASCEND_DEVICE_ID=$RANK_ID + echo "Device ID: $RANK_IDS" + # export RANK_IDS=$RANK_IDS + export ASCEND_DEVICE_ID=$RANK_IDS + ASCEND_DEVICE_ID=$RANK_IDS export DEVICE_ID=$ASCEND_DEVICE_ID DEVICE_INDEX=$(( DEVICE_ID + RANK_INDEX * 8 )) export DEVICE_INDEX=$DEVICE_INDEX @@ -128,8 +139,8 @@ do #执行训练脚本,需要模型审视修改 corenum=`cat /proc/cpuinfo |grep 'processor' |wc -l` - let a=RANK_ID*${corenum}/8 - let b=RANK_ID+1 + let a=RANK_IDS*${corenum}/8 + let b=RANK_IDS+1 let c=b*${corenum}/8-1 if [ "x${bind_core}" != x ];then bind_core="taskset -c $a-$c" @@ -158,6 +169,7 @@ wait #训练结束时间,不需要修改 end_time=$(date +%s) e2e_time=$(( $end_time - $start_time )) +sed -i 's/RANK_SIZES/RANK_SIZE/g' src/data_loader/resnet50/data_loader.py echo "------------------ Final result ------------------" #单step时长,需要从train_$ASCEND_DEVICE_ID.log里,通过关键字获取。需要模型审视修改 @@ -174,13 +186,13 @@ echo "E2E Training Duration sec : $e2e_time" #训练用例信息,不需要修改 BatchSize=${batch_size} DeviceType=`uname -m` -CaseName=${Network}${name_bind}_bs${BatchSize}_${RANK_SIZE}'p'_'acc' +CaseName=${Network}${name_bind}_bs${BatchSize}_${RANK_SIZES}'p'_'acc' ##获取性能数据 #吞吐量,不需要修改 ActualFPS=${FPS} #单迭代训练时长,不需要修改 -TrainingTime=`awk 'BEGIN{printf "%.2f\n",'${BatchSize}'*'${RANK_SIZE}'*1000/'${FPS}'}'` +TrainingTime=`awk 'BEGIN{printf "%.2f\n",'${BatchSize}'*'${RANK_SIZES}'*1000/'${FPS}'}'` ##获取性能数据,不需要修改 @@ -197,7 +209,7 @@ ActualLoss=`awk 'END {print}' $cur_path/output/$ASCEND_DEVICE_ID/train_${CaseNam #关键信息打印到${CaseName}.log中,不需要修改 echo "Network = ${Network}" > $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log -echo "RankSize = ${RANK_SIZE}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZES}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "BatchSize = ${BatchSize}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "DeviceType = ${DeviceType}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "CaseName = ${CaseName}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log @@ -205,4 +217,4 @@ echo "ActualFPS = ${ActualFPS}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log -echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log \ No newline at end of file +echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log diff --git a/TensorFlow/built-in/cv/image_segmentation/UNet3D_ID0057_for_TensorFlow/README.md b/TensorFlow/built-in/cv/image_segmentation/UNet3D_ID0057_for_TensorFlow/README.md index 1209d2ef91a4b327ceadf1b06792792ad8b1b754..b13088414c2134693db7f8241d8ee0b416723a48 100644 --- a/TensorFlow/built-in/cv/image_segmentation/UNet3D_ID0057_for_TensorFlow/README.md +++ b/TensorFlow/built-in/cv/image_segmentation/UNet3D_ID0057_for_TensorFlow/README.md @@ -181,6 +181,8 @@ run_config = NPURunConfig( bash run_accuracy_8p.sh --data_dir=/data/BraTS2019Train ``` + 环境变量方式拉起详情请参考: + https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/63RC1alpha002/tfmoddevg/tfmigr1/atlasmprtg_13_9036.html ## 高级参考 #### 脚本和示例代码 diff --git a/TensorFlow/built-in/cv/image_segmentation/UNet3D_ID0057_for_TensorFlow/scripts/run_accuracy_8p.sh b/TensorFlow/built-in/cv/image_segmentation/UNet3D_ID0057_for_TensorFlow/scripts/run_accuracy_8p.sh index 771f5fb921c1d2f0df9f4bca94c0293c7172f86a..95427b89cb5ecd46efefe2742bcd1ec0655fbce0 100644 --- a/TensorFlow/built-in/cv/image_segmentation/UNet3D_ID0057_for_TensorFlow/scripts/run_accuracy_8p.sh +++ b/TensorFlow/built-in/cv/image_segmentation/UNet3D_ID0057_for_TensorFlow/scripts/run_accuracy_8p.sh @@ -9,8 +9,8 @@ mkdir -p ${upDir}/test/output/0 # user env export JOB_ID=NPU20210126 -export RANK_SIZE=8 -export RANK_TABLE_FILE=${currentDir}/8p.json +export RANK_SIZES=8 +#export RANK_TABLE_FILE=${currentDir}/8p.json data_dir=$1 fold=$2 @@ -21,7 +21,7 @@ if [ x"${fold}" = x"all" ] ; then for device_index in ${device_group} do - RANK_ID=${device_index} ASCEND_DEVICE_ID=${device_index} ${currentDir}/train_accuracy_8p.sh ${data_dir} 0 & + RANK_IDS=${device_index} ASCEND_DEVICE_ID=${device_index} ${currentDir}/train_accuracy_8p.sh ${data_dir} 0 & done wait @@ -32,7 +32,7 @@ else echo "[`date +%Y%m%d-%H:%M:%S`] [INFO] fold$fold train start" for device_index in ${device_group} do - RANK_ID=${device_index} ASCEND_DEVICE_ID=${device_index} ${currentDir}/train_accuracy_8p.sh ${data_dir} ${fold} & + RANK_IDS=${device_index} ASCEND_DEVICE_ID=${device_index} ${currentDir}/train_accuracy_8p.sh ${data_dir} ${fold} & done wait diff --git a/TensorFlow/built-in/cv/image_segmentation/UNet3D_ID0057_for_TensorFlow/test/train_full_8p.sh b/TensorFlow/built-in/cv/image_segmentation/UNet3D_ID0057_for_TensorFlow/test/train_full_8p.sh index 02a6ba19f7063dd1c47bedf34e76ffa88dc2f538..b69de76e242e2f58effe146caae2f1b60358e16c 100644 --- a/TensorFlow/built-in/cv/image_segmentation/UNet3D_ID0057_for_TensorFlow/test/train_full_8p.sh +++ b/TensorFlow/built-in/cv/image_segmentation/UNet3D_ID0057_for_TensorFlow/test/train_full_8p.sh @@ -5,9 +5,9 @@ cur_path=`pwd` export LANG=en_US.UTF-8 -export RANK_SIZE=8 +export RANK_SIZES=8 export JOB_ID=10087 -export RANK_TABLE_FILE=$cur_path/../scripts/8p.json +#export RANK_TABLE_FILE=$cur_path/../scripts/8p.json RANK_ID_START=0 @@ -71,9 +71,19 @@ do cp -rf $install_path/fwkacllib/data/rl/Ascend910/custom ${autotune_dump_path}/RL/ elif [[ $para == --data_path* ]];then data_path=`echo ${para#*=}` + elif [[ $para == --one_node_ip* ]];then + one_node_ip=`echo ${para#*=}` fi done +#8p训练必须参数(本机IP) +one_node_ip=$one_node_ip +#新增适配集群环境变量 +export CM_CHIEF_IP=${one_node_ip} #主节点ip,所有服务器一致 +export CM_CHIEF_PORT=29688 #通信端口,所有服务器一致 +export CM_CHIEF_DEVICE=0 #配置为0,配置主卡,类似于主节点,所有服务器一致 +export CM_WORKER_SIZE=8 #卡数,单机为8,所有服务器一致 +export CM_WORKER_IP=${one_node_ip} #当前服务器ip,不同环境ip不同 #data_path='../' #校验是否传入data_path,不需要修改 @@ -81,18 +91,23 @@ if [[ $data_path == "" ]];then echo "[Error] para \"data_path\" must be confing" exit 1 fi - +cd $cur_path/../ +sed -i 's/RANK_SIZE/RANK_SIZES/g' model/model_fn.py pbinference/unet3d_pb_inference.sh main_npu.py dataset/data_loader.py runtime/hooks.py runtime/setup.py +sed -i 's/RANK_ID/RANK_IDS/g' pbinference/unet3d_pb_inference.sh main_npu.py dataset/data_loader.py runtime/hooks.py runtime/setup.py cd $cur_path/../scripts #训练开始时间,不需要修改 start_time=$(date +%s) bash run_accuracy_8p.sh ${data_path} all +wait #训练结束时间,不需要修改 end_time=$(date +%s) e2e_time=$(( $end_time - $start_time )) - +cd $cur_path/../ +sed -i 's/RANK_SIZES/RANK_SIZE/g' model/model_fn.py pbinference/unet3d_pb_inference.sh main_npu.py dataset/data_loader.py runtime/hooks.py runtime/setup.py +sed -i 's/RANK_IDS/RANK_ID/g' pbinference/unet3d_pb_inference.sh main_npu.py dataset/data_loader.py runtime/hooks.py runtime/setup.py sleep 30 train_accuracy=`grep -r "whole" $cur_path/output/0/train_0.log | awk '{print $6}'` @@ -105,7 +120,6 @@ echo "E2E Training Duration sec : $e2e_time" echo "------------------ Final result ------------------" #输出性能FPS,需要模型审视修改 fps=`grep throughput_train $cur_path/output/0/train_0.log|awk -F 'throughput_train' '{print $2}'|awk -F ':' '{print $2}'|awk '{print $1}'` -#FPS=`awk 'BEGIN{printf "%.2f\n",'${RANK_SIZE}'*'${fps}'}'` FPS=1.5 #打印,不需要修改 echo "Final Performance images/sec : $FPS" @@ -116,13 +130,13 @@ echo "E2E Training Duration sec : $e2e_time" #训练用例信息,不需要修改 BatchSize=${batch_size} DeviceType=`uname -m` -CaseName=${Network}_bs${BatchSize}_${RANK_SIZE}'p'_'acc' +CaseName=${Network}_bs${BatchSize}_${RANK_SIZES}'p'_'acc' #获取性能数据,不需要修改 #吞吐量 ActualFPS=${FPS} #单迭代训练时长 -TrainingTime=`awk 'BEGIN{printf "%.2f\n",'${BatchSize}'*'${RANK_SIZE}'*1000/'${FPS}'}'` +TrainingTime=`awk 'BEGIN{printf "%.2f\n",'${BatchSize}'*'${RANK_SIZES}'*1000/'${FPS}'}'` #从train_$ASCEND_DEVICE_ID.log提取Loss到train_${CaseName}_loss.txt中,需要根据模型审视 @@ -133,7 +147,7 @@ ActualLoss=`awk 'END {print}' $cur_path/output/0/train_${CaseName}_loss.txt` #关键信息打印到${CaseName}.log中,不需要修改 echo "Network = ${Network}" > $cur_path/output/0/${CaseName}.log -echo "RankSize = ${RANK_SIZE}" >> $cur_path/output/0/${CaseName}.log +echo "RankSize = ${RANK_SIZES}" >> $cur_path/output/0/${CaseName}.log echo "BatchSize = ${BatchSize}" >> $cur_path/output/0/${CaseName}.log echo "DeviceType = ${DeviceType}" >> $cur_path/output/0/${CaseName}.log echo "CaseName = ${CaseName}" >> $cur_path/output/0/${CaseName}.log diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/README.md b/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/README.md index c9e116c76e1d5b02ae8b1bbb145b9dd980d30dc8..d506b9e46a30f7651341afbbd002a43225f3a2b4 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/README.md +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/README.md @@ -118,7 +118,15 @@ pip3 install requirements.txt bash train_performance_1p.sh --data_path=/data (功能和性能) bash train_full_1p.sh --data_path=/data (全量) ``` +- 8卡训练-CMxxx环境变量方式拉起 + 以数据集放在/data为例 + +环境变量方式拉起详情请参考: +https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/63RC1alpha002/tfmoddevg/tfmigr1/atlasmprtg_13_9036.html + + bash train_performance_8p.sh --data_path=/data --one_node_ip=本机IP + bash train_full_8p.sh --data_path=/data --one_node_ip=本机IP ## 高级参考 #### 脚本和示例代码 diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_full_8p.sh b/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_full_8p.sh index e58e09555687d14d31fe22af0948add099c5f108..7250053b376a4ff13fa825c662658b18fab2bd60 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_full_8p.sh +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_full_8p.sh @@ -4,13 +4,11 @@ cur_path=`pwd` #集合通信参数,不需要修改 - - export HCCL_CONNECT_TIMEOUT=1200 #集合通信参数,不需要修改 -export RANK_SIZE=8 -export RANK_TABLE_FILE=$cur_path/8p.json +export RANK_SIZES=8 +#export RANK_TABLE_FILE=$cur_path/8p.json export JOB_ID=10087 RANK_ID_START=0 ASCEND_DEVICE_ID_START=0 @@ -57,14 +55,19 @@ elif [[ $para == --over_dump* ]];then mkdir -p ${over_dump_path} elif [[ $para == --data_path* ]];then data_path=`echo ${para#*=}` + elif [[ $para == --one_node_ip* ]];then + one_node_ip=`echo ${para#*=}` fi done -#校验是否传入data_path,不需要修改 -if [[ $data_path == "" ]];then - echo "[Error] para \"data_path\" must be confing" - exit 1 -fi +#8p训练必须参数(本机IP) +one_node_ip=$one_node_ip +#新增适配集群环境变量 +export CM_CHIEF_IP=${one_node_ip} #主节点ip,所有服务器一致 +export CM_CHIEF_PORT=29688 #通信端口,所有服务器一致 +export CM_CHIEF_DEVICE=0 #配置为0,配置主卡,类似于主节点,所有服务器一致 +export CM_WORKER_SIZE=8 #卡数,单机为8,所有服务器一致 +export CM_WORKER_IP=${one_node_ip} #当前服务器ip,不同环境ip不同 #校验是否传入data_path,不需要修改 if [[ $data_path == "" ]];then @@ -102,6 +105,8 @@ sed -i "s%/npu/traindata/ID2940_CarPeting_TF_WideDeep_TF%${data_path}%p" configs sed -i "s%./model%$cur_path/output/$ASCEND_DEVICE_ID/ckpt%p" configs/config.py sed -i "s%display_step = 100%display_step = $display_step%p" configs/config.py sed -i "s%n_epoches = 2%n_epoches = $n_epoches%p" configs/config.py +sed -i 's/RANK_SIZE/RANK_SIZES/g' widedeep/WideDeep_fp16_huifeng.py +sed -i 's/RANK_SIZE/RANK_SIZES/g' train.py #echo `cat configs/config.py |uniq > configs/config.py; cp -f configs/config.py configs/config.py.run` cp configs/config.py configs/config.py.run @@ -109,13 +114,13 @@ cp configs/config.py configs/config.py.run cd $cur_path/../ start=$(date +%s) -for((RANK_ID=$RANK_ID_START;RANK_ID<$((RANK_SIZE+RANK_ID_START));RANK_ID++)); +for((RANK_IDS=$RANK_ID_START;RANK_IDS<$((RANK_SIZES+RANK_ID_START));RANK_IDS++)); do #设置环境变量,不需要修改 - echo "Device ID: $RANK_ID" - export RANK_ID=$RANK_ID - export ASCEND_DEVICE_ID=$RANK_ID - ASCEND_DEVICE_ID=$RANK_ID + echo "Device ID: $RANK_IDS" + export RANK_IDS=$RANK_IDS + export ASCEND_DEVICE_ID=$RANK_IDS + ASCEND_DEVICE_ID=$RANK_IDS if [ -d $cur_path/output/${ASCEND_DEVICE_ID} ];then rm -rf $cur_path/output/${ASCEND_DEVICE_ID} mkdir -p $cur_path/output/${ASCEND_DEVICE_ID} @@ -131,7 +136,8 @@ done wait end=$(date +%s) e2e_time=$(( $end - $start )) - +sed -i 's/RANK_SIZES/RANK_SIZE/g' train.py +sed -i 's/RANK_SIZES/RANK_SIZE/g' widedeep/WideDeep_fp16_huifeng.py #配置文件恢复 mv -f configs/config.py.bak configs/config.py @@ -155,9 +161,9 @@ echo "E2E Training Duration sec : $e2e_time" BatchSize=${batch_size} DeviceType=`uname -m` if [[ $precision_mode == "must_keep_origin_dtype" ]];then - CaseName=${Network}_bs${BatchSize}_${RANK_SIZE}'p'_'fp32'_'acc' + CaseName=${Network}_bs${BatchSize}_${RANK_SIZES}'p'_'fp32'_'acc' else - CaseName=${Network}_bs${BatchSize}_${RANK_SIZE}'p'_'acc' + CaseName=${Network}_bs${BatchSize}_${RANK_SIZES}'p'_'acc' fi ##获取性能数据 @@ -176,7 +182,7 @@ ActualLoss=`cat $cur_path/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt | #关键信息打印到${CaseName}.log中,不需要修改 echo "Network = ${Network}" > $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log -echo "RankSize = ${RANK_SIZE}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZES}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "BatchSize = ${BatchSize}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "DeviceType = ${DeviceType}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "CaseName = ${CaseName}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log