From 6f260192c71b6edc12b045010378aa63d038e6ca Mon Sep 17 00:00:00 2001 From: ykxia Date: Fri, 11 Nov 2022 11:32:00 +0800 Subject: [PATCH 1/2] =?UTF-8?q?Transformer=5FID0633=5Ffor=5FTensorFlow2.X?= =?UTF-8?q?=E4=BD=BF=E8=83=BDRT2.0+=E4=BA=8C=E8=BF=9B=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...T2_performance_1p_4096bs_dynamic_noeval.sh | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 TensorFlow2/built-in/nlp/Transformer_ID0633_for_TensorFlow2.X/test/train_RT2_performance_1p_4096bs_dynamic_noeval.sh diff --git a/TensorFlow2/built-in/nlp/Transformer_ID0633_for_TensorFlow2.X/test/train_RT2_performance_1p_4096bs_dynamic_noeval.sh b/TensorFlow2/built-in/nlp/Transformer_ID0633_for_TensorFlow2.X/test/train_RT2_performance_1p_4096bs_dynamic_noeval.sh new file mode 100644 index 000000000..6db754592 --- /dev/null +++ b/TensorFlow2/built-in/nlp/Transformer_ID0633_for_TensorFlow2.X/test/train_RT2_performance_1p_4096bs_dynamic_noeval.sh @@ -0,0 +1,200 @@ +#!/bin/bash + +#当前路径,不需要修改 +cur_path=`pwd` + +#集合通信参数,不需要修改 + +export RANK_SIZE=1 +export JOB_ID=10087 +export RANK_ID_START=0 +export PYTHONPATH=../transformer:$PYTHONPATH + +#使能RT2.0 +export ENABLE_RUNTIME_V2=1 + +# 数据集路径,保持为空,不需要修改 +data_path="" + +#设置默认日志级别,不需要修改 + +#基础参数，需要模型审视修改 +#网络名称，同目录名称 +Network="Transformer_ID0633_for_TensorFlow2.X" +#训练batch_size +batch_size=4096 +#训练step +train_steps=300 + +#TF2.X独有，不需要修改 +#export NPU_ENABLE_PERF=true + +#维测参数，precision_mode需要模型审视修改 +precision_mode="allow_mix_precision" +#维持参数，以下不需要修改 +over_dump=False +data_dump_flag=False +data_dump_step="10" +profiling=False + + +# 帮助信息，不需要修改 +if [[ $1 == --help || $1 == -h ]];then + echo"usage:./train_full_1p.sh " + echo " " + echo "parameter explain: + --precision_mode precision mode(allow_fp32_to_fp16/force_fp16/must_keep_origin_dtype/allow_mix_precision) + --over_dump if or not over detection, default is False + --data_dump_flag data dump flag, default is False + --data_dump_step data dump step, default is 10 + --profiling if or not profiling for performance debug, default is False + --data_path source data of training + -h/--help show help message + " + exit 1 +fi + +#参数校验，不需要修改 +for para in $* +do + if [[ $para == --precision_mode* ]];then + precision_mode=`echo ${para#*=}` + elif [[ $para == --over_dump* ]];then + over_dump=`echo ${para#*=}` + over_dump_path=${cur_path}/output/overflow_dump + mkdir -p ${over_dump_path} + elif [[ $para == --data_dump_flag* ]];then + data_dump_flag=`echo ${para#*=}` + data_dump_path=${cur_path}/output/data_dump + mkdir -p ${data_dump_path} + elif [[ $para == --data_dump_step* ]];then + data_dump_step=`echo ${para#*=}` + elif [[ $para == --profiling* ]];then + profiling=`echo ${para#*=}` + profiling_dump_path=${cur_path}/output/profiling + mkdir -p ${profiling_dump_path} + elif [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + fi +done + +#校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be confing" + exit 1 +fi + +#训练开始时间，不需要修改 +start_time=$(date +%s) + +#进入训练脚本目录，需要模型审视修改 + +for((RANK_ID=$RANK_ID_START;RANK_ID<$((RANK_SIZE+RANK_ID_START));RANK_ID++)); +do + #设置环境变量，不需要修改 + echo "Device ID: $ASCEND_DEVICE_ID" + export RANK_ID=$RANK_ID + + + + #创建DeviceID输出目录，不需要修改 + if [ -d ${cur_path}/output/${ASCEND_DEVICE_ID} ];then + rm -rf ${cur_path}/output/${ASCEND_DEVICE_ID} + mkdir -p ${cur_path}/output/$ASCEND_DEVICE_ID/ckpt_${learning_rate} + else + mkdir -p ${cur_path}/output/$ASCEND_DEVICE_ID/ckpt_${learning_rate} + fi + + #绑核，不需要绑核的模型删除，需要绑核的模型根据实际修改 + cpucount=`lscpu | grep "CPU(s):" | head -n 1 | awk '{print $2}'` + cpustep=`expr $cpucount / 8` + echo "taskset c steps:" $cpustep + let a=RANK_ID*$cpustep + let b=RANK_ID+1 + let c=b*$cpustep-1 + + #执行训练脚本，以下传参不需要修改，其他需要模型审视修改 + #--data_dir, --model_dir, --precision_mode, --over_dump, --over_dump_path，--data_dump_flag，--data_dump_step，--data_dump_path，--profiling，--profiling_dump_path，--autotune + nohup taskset -c $a-$c python3 ../transformer/official/nlp/transformer/transformer_main.py \ + --data_dir=${data_path} \ + --model_dir=${cur_path}/output/$ASCEND_DEVICE_ID/ckpt \ + --vocab_file=${data_path}/vocab.ende.32768 \ + --param_set=big \ + --train_steps=${train_steps} \ + --batch_size=${batch_size} \ + --steps_between_evals=100 \ + --max_length=64 \ + --mode=train \ + --decode_batch_size=32 \ + --decode_max_length=97 \ + --padded_decode=False \ + --num_gpus=1 \ + --dtype=fp16 \ + --distribution_strategy='one_device' \ + --enable_time_history=true \ + --log_steps=100 \ + --loss_scale='dynamic' \ + --precision_mode=${precision_mode} \ + --over_dump=${over_dump} \ + --over_dump_path=${over_dump_path} \ + --data_dump_flag=${data_dump_flag} \ + --data_dump_step=${data_dump_step} \ + --data_dump_path=${data_dump_path} \ + --profiling=${profiling} \ + --profiling_dump_path=${profiling_dump_path} > ${cur_path}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log 2>&1 & +done +wait + +#训练结束时间，不需要修改 +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + +#############冒烟看护######################### +BatchSize=${batch_size} +#设备类型 +DeviceType=`uname -m` +#用例名称 +CaseName=${Network}_bs${BatchSize}_${RANK_SIZE}'p'_'RT2'_'perf' + +echo "------------------ Final result ------------------" +#输出性能FPS，需要模型审视修改 +single_batch_step_sec=`grep TimeHistory $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|awk 'END {print $8}'` +FPS=`awk 'BEGIN{printf "%.2f\n",'${single_batch_step_sec}'}'` +#打印，不需要修改 +echo "Final Performance images/sec : $FPS" + +grep "Train history" $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log|awk '{print$8}'|sed 's/,//g'|sed 's/\[//g'|sed 's/\]//g'|sed 's/\}//g' >> $cur_path/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt +ActualLoss=`awk 'END {print}' $cur_path/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt` +#输出训练精度,需要模型审视修改 +grep "Train history" $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log|awk '{print$8}'|sed 's/,//g'|sed 's/\[//g'|sed 's/\]//g'|sed 's/\}//g' >> $cur_path/output/$ASCEND_DEVICE_ID/train_${CaseName}_acc.txt +train_accuracy=`awk 'END {print}' $cur_path/output/$ASCEND_DEVICE_ID/train_${CaseName}_acc.txt` +#train_accuracy=`grep eval_accuracy $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|grep -v mlp_log|awk 'END {print $5}'|sed 's/,//g'|cut -c 1-5` +#打印，不需要修改 +echo "Final Train Accuracy : ${train_accuracy}" +echo "E2E Training Duration sec : $e2e_time" + +##获取性能数据 +#吞吐量，不需要修改 +ActualFPS=${FPS} +#单迭代训练时长，不需要修改 +TrainingTime=`awk 'BEGIN{printf "%.2f\n",'${BatchSize}'*'${RANK_SIZE}'*1000/'${FPS}'}'` + +##获取Loss +#从train_$ASCEND_DEVICE_ID.log提取Loss到train_${CaseName}_loss.txt中 +#grep loss $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log|awk '{print$11}'|grep -v instead|grep -v masked_lm_loss|sed 's/,//g'|sed '/^$/d' >> $cur_path/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt + +#最后一个迭代loss值 +#ActualLoss=`awk 'END {print}' $cur_path/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt` + +#关键信息打印到${CaseName}.log中 +echo "Network = ${Network}" > $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${BatchSize}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = ${DeviceType}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualFPS = ${ActualFPS}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log + -- Gitee From 03b332cdc6b3a0d5f7ab31e1be8abc39e8805c07 Mon Sep 17 00:00:00 2001 From: ykxia Date: Fri, 11 Nov 2022 13:39:22 +0800 Subject: [PATCH 2/2] =?UTF-8?q?Transformer=5FID3215=5Ffor=5FTensorFlow2.X?= =?UTF-8?q?=E4=BD=BF=E8=83=BDRT2.0=E4=BA=8C=E8=BF=9B=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...h => train_RT2_performance_1p_6144bs_dynamic_noeval.sh} | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) rename TensorFlow2/built-in/nlp/Transformer_ID0633_for_TensorFlow2.X/test/{train_RT2_performance_1p_4096bs_dynamic_noeval.sh => train_RT2_performance_1p_6144bs_dynamic_noeval.sh} (98%) diff --git a/TensorFlow2/built-in/nlp/Transformer_ID0633_for_TensorFlow2.X/test/train_RT2_performance_1p_4096bs_dynamic_noeval.sh b/TensorFlow2/built-in/nlp/Transformer_ID0633_for_TensorFlow2.X/test/train_RT2_performance_1p_6144bs_dynamic_noeval.sh similarity index 98% rename from TensorFlow2/built-in/nlp/Transformer_ID0633_for_TensorFlow2.X/test/train_RT2_performance_1p_4096bs_dynamic_noeval.sh rename to TensorFlow2/built-in/nlp/Transformer_ID0633_for_TensorFlow2.X/test/train_RT2_performance_1p_6144bs_dynamic_noeval.sh index 6db754592..fc5e7082d 100644 --- a/TensorFlow2/built-in/nlp/Transformer_ID0633_for_TensorFlow2.X/test/train_RT2_performance_1p_4096bs_dynamic_noeval.sh +++ b/TensorFlow2/built-in/nlp/Transformer_ID0633_for_TensorFlow2.X/test/train_RT2_performance_1p_6144bs_dynamic_noeval.sh @@ -20,14 +20,14 @@ data_path="" #基础参数，需要模型审视修改 #网络名称，同目录名称 -Network="Transformer_ID0633_for_TensorFlow2.X" +Network="Transformer_ID3215_for_TensorFlow2.X" #训练batch_size -batch_size=4096 +batch_size=6144 #训练step train_steps=300 #TF2.X独有，不需要修改 -#export NPU_ENABLE_PERF=true +export NPU_ENABLE_PERF=true #维测参数，precision_mode需要模型审视修改 precision_mode="allow_mix_precision" @@ -197,4 +197,3 @@ echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${Ca echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log - -- Gitee