diff --git a/TensorFlow/built-in/cv/detection/MaskRcnn_ID0011_for_TensorFlow/test/train_RT2_performance_1p.sh b/TensorFlow/built-in/cv/detection/MaskRcnn_ID0011_for_TensorFlow/test/train_RT2_performance_1p.sh index 3f0ce9cc497e67ba3663906d0c8efcae721d8d11..e2b7d999bd6c29da09f3363986b0a8bdb69e54bb 100644 --- a/TensorFlow/built-in/cv/detection/MaskRcnn_ID0011_for_TensorFlow/test/train_RT2_performance_1p.sh +++ b/TensorFlow/built-in/cv/detection/MaskRcnn_ID0011_for_TensorFlow/test/train_RT2_performance_1p.sh @@ -187,6 +187,9 @@ FPS=`awk 'BEGIN{printf "%f\n",'${batch_size}'*'${RANK_SIZE}'*'${FPSper}'}'` #打印,不需要修改 echo "Final Performance images/sec : $FPS" +#输出CompileTime +CompileTime=`grep "INFO:tensorflow:loss" $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log |head -n 2|awk '{if (NR>1) print $7}' | awk -F '(' '{print $2}'` + #输出训练精度,需要模型审视修改 train_accuracy=`grep "Average Precision" $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|head -1|awk '{print $13}'` #打印,不需要修改 @@ -222,3 +225,4 @@ echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${Ca #echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CompileTime = ${CompileTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log diff --git a/TensorFlow/built-in/cv/image_classification/PixelLink_ID3056_for_TensorFlow/test/train_RT2_performance_1p.sh b/TensorFlow/built-in/cv/image_classification/PixelLink_ID3056_for_TensorFlow/test/train_RT2_performance_1p.sh index e6ec771efb2b05d454d1d4aeb63bee5b3f673d8b..2ebb42aa736339c8883adb9acb81ce55c595a69c 100644 --- a/TensorFlow/built-in/cv/image_classification/PixelLink_ID3056_for_TensorFlow/test/train_RT2_performance_1p.sh +++ b/TensorFlow/built-in/cv/image_classification/PixelLink_ID3056_for_TensorFlow/test/train_RT2_performance_1p.sh @@ -136,6 +136,8 @@ FPS=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'/'${TrainingTime}'}'` #打印,不需要修改 echo "Final Performance item/sec : $FPS" +#输出CompileTime +CompileTime=`grep "sec/step" $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log |head -n 1|awk '{print $7}' | awk -F '(' '{print $2}'` # #输出训练精度,需要模型审视修改 #train_accuracy=`grep "test AUC" ${cur_path}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|awk '{print $3}'` @@ -169,3 +171,4 @@ echo "ActualFPS = ${ActualFPS}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CompileTime = ${CompileTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log diff --git a/TensorFlow/built-in/nlp/CNN-CTC_ID0683_for_TensorFlow/main_rt.py b/TensorFlow/built-in/nlp/CNN-CTC_ID0683_for_TensorFlow/main_rt.py index b85b570532b084509869e78ee5843a0d5314a9da..701ac1dddc4ae94fa59c5289dc3da70a549846b9 100644 --- a/TensorFlow/built-in/nlp/CNN-CTC_ID0683_for_TensorFlow/main_rt.py +++ b/TensorFlow/built-in/nlp/CNN-CTC_ID0683_for_TensorFlow/main_rt.py @@ -105,7 +105,7 @@ def train(train_dir=None, val_dir=None, mode='train'): # the training part for cur_batch in range(num_batches_per_epoch): - if (cur_batch + 1) % 100 == 0: + if (cur_batch + 1) % 100 == 0 or cur_batch <= 5: print('batch', cur_batch, ': time', time.time() - batch_time) batch_time = time.time() indexs = [shuffle_idx[i % num_train_samples] for i in diff --git a/TensorFlow/built-in/nlp/CNN-CTC_ID0683_for_TensorFlow/test/train_RT2_performance_1p.sh b/TensorFlow/built-in/nlp/CNN-CTC_ID0683_for_TensorFlow/test/train_RT2_performance_1p.sh index ed46648fd31f67864388d70a0077b2ac25a47d70..d613f961f612dbb388fe6f1d6d4f9d0c58987aba 100644 --- a/TensorFlow/built-in/nlp/CNN-CTC_ID0683_for_TensorFlow/test/train_RT2_performance_1p.sh +++ b/TensorFlow/built-in/nlp/CNN-CTC_ID0683_for_TensorFlow/test/train_RT2_performance_1p.sh @@ -69,6 +69,9 @@ FPS=`awk 'BEGIN{printf "%.2f\n",'${batch_size}'/'${TrainingTime}'}'` #打印,不需要修改 echo "Final Performance images/sec : $FPS" +#输出CompileTime +CompileTime=`grep "time" $cur_path/test/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log | grep batch |head -n 3|awk '{sum+=$5} END {print sum}'` + #输出训练精度,需要模型审视修改 train_accuracy=`grep "accuracy" $cur_path/test/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log|awk 'END {print $7}'|cut -d , -f 1` #打印,不需要修改 @@ -103,3 +106,4 @@ echo "TrainingTime = ${TrainingTime}" >> $cur_path/test/output/$ASCEND_DEVICE_ID echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "ActualLoss = ${ActualLoss}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CompileTime = ${CompileTime}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log diff --git a/TensorFlow/built-in/nlp/DS-CNN_RT2_ID1769_for_TensorFlow/test/train_RT2_performance_1p.sh b/TensorFlow/built-in/nlp/DS-CNN_RT2_ID1769_for_TensorFlow/test/train_RT2_performance_1p.sh index 3cc15b418be92d6dfa9be051b2978b2e38ab314a..e7f13c7254dd03b06094034949751257d8154084 100644 --- a/TensorFlow/built-in/nlp/DS-CNN_RT2_ID1769_for_TensorFlow/test/train_RT2_performance_1p.sh +++ b/TensorFlow/built-in/nlp/DS-CNN_RT2_ID1769_for_TensorFlow/test/train_RT2_performance_1p.sh @@ -172,6 +172,9 @@ FPS=`awk 'BEGIN{printf "%.2f\n",'${batch_size}'*'${step_per_s}'}'` #打印,不需要修改 echo "Final Performance images/sec : $FPS" +#输出CompileTime +CompileTime=`grep 'train duration:' $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log |head -n 1|awk '{print $(NF)}'` + #输出训练精度,需要模型审视修改 train_accuracy=`grep 'INFO:tensorflow.*Validation accuracy' $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log|awk 'END {print $6}'|awk -F % 'END {print $1}'` train_accuracy=`awk 'BEGIN{printf "%.4f\n",'${train_accuracy}'/100}'` @@ -208,3 +211,4 @@ echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/ echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CompileTime = ${CompileTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log diff --git a/TensorFlow/built-in/nlp/Siamese_ID0506_for_TensorFlow/test/train_RT2_performance_1p.sh b/TensorFlow/built-in/nlp/Siamese_ID0506_for_TensorFlow/test/train_RT2_performance_1p.sh index 2a5cb1c2b79f993ed17b5a6ae9e28fc167fc2eb9..7877d6fe9888c94c323389e6e7ff93810ea798fd 100644 --- a/TensorFlow/built-in/nlp/Siamese_ID0506_for_TensorFlow/test/train_RT2_performance_1p.sh +++ b/TensorFlow/built-in/nlp/Siamese_ID0506_for_TensorFlow/test/train_RT2_performance_1p.sh @@ -87,6 +87,8 @@ train_accuracy=`grep "TRAIN " $cur_path/test/output/$ASCEND_DEVICE_ID/train_$ASC #打印,不需要修改 #echo "Final Train Accuracy : ${train_accuracy}" +#输出CompileTime +CompileTime=`grep "time(ms)" $cur_path/test/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log |head -n 1|awk '{print $6/1000}'` #性能看护结果汇总 #训练用例信息,不需要修改 @@ -117,3 +119,4 @@ echo "TrainingTime = ${TrainingTime}" >> $cur_path/test/output/$ASCEND_DEVICE_ID echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "ActualLoss = ${ActualLoss}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CompileTime = ${CompileTime}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log diff --git a/TensorFlow/built-in/recommendation/DIEN_ID3065_for_TensorFlow/script/train.py b/TensorFlow/built-in/recommendation/DIEN_ID3065_for_TensorFlow/script/train.py index 8f99a5799d13b410c9cb17fba1d7624a1f8c5c2e..d866126f1208b24ea52f4701507397c608444247 100644 --- a/TensorFlow/built-in/recommendation/DIEN_ID3065_for_TensorFlow/script/train.py +++ b/TensorFlow/built-in/recommendation/DIEN_ID3065_for_TensorFlow/script/train.py @@ -233,7 +233,7 @@ def train( aux_loss_sum += aux_loss iter += 1 sys.stdout.flush() - if (iter % test_iter) == 0: + if (iter % test_iter) == 0 or iter <= 5: avg_examples_per_second = batch_size/(end_time - start_time) print("avg_examples_per_second: ", avg_examples_per_second) print('iter: %d ----> train_loss: %.4f ---- train_accuracy: %.4f ---- train_aux_loss: %.4f ---- perf: %.4f' % \ diff --git a/TensorFlow/built-in/recommendation/DIEN_ID3065_for_TensorFlow/test/train_RT2_performance_1p.sh b/TensorFlow/built-in/recommendation/DIEN_ID3065_for_TensorFlow/test/train_RT2_performance_1p.sh index a397bc8f79f3f8c064d3fd01f13ffe1fab3e8987..b0ad15296c1cb932a43588bccbdc7fe3a31b6416 100644 --- a/TensorFlow/built-in/recommendation/DIEN_ID3065_for_TensorFlow/test/train_RT2_performance_1p.sh +++ b/TensorFlow/built-in/recommendation/DIEN_ID3065_for_TensorFlow/test/train_RT2_performance_1p.sh @@ -135,6 +135,9 @@ FPS=`grep avg_examples_per_second $cur_path/output/$ASCEND_DEVICE_ID/train_$ASC #打印,不需要修改 echo "Final Performance item/sec : $FPS" +#输出CompileTime +CompileTime=`grep "perf:" $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log |head -n 1|awk '{print $14}'` + #输出训练精度,需要模型审视修改 train_accuracy=`grep "train_accuracy" $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log | awk -F "train_accuracy:" 'END{print $2}' | awk -F ' ' '{print $1}'|sed s/[[:space:]]//g` #打印,不需要修改 @@ -170,4 +173,5 @@ echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/output/$ASCEND_DEVICE_ID/$ echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CompileTime = ${CompileTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log diff --git a/TensorFlow/built-in/recommendation/DeepCTR_Series_for_TensorFlow/test/train_ID3058_MMoE_performance_1p_RT2.sh b/TensorFlow/built-in/recommendation/DeepCTR_Series_for_TensorFlow/test/train_ID3058_MMoE_performance_1p_RT2.sh index 0063fcacb024c3c498781b5dabbe04729ca29856..91918048a6e452a3111763aedb2e3fb4090bdea3 100644 --- a/TensorFlow/built-in/recommendation/DeepCTR_Series_for_TensorFlow/test/train_ID3058_MMoE_performance_1p_RT2.sh +++ b/TensorFlow/built-in/recommendation/DeepCTR_Series_for_TensorFlow/test/train_ID3058_MMoE_performance_1p_RT2.sh @@ -125,6 +125,8 @@ FPS=`awk 'BEGIN{printf "%.2f\n", 1 /'${Time}'*1000000}'` #打印,不需要修改 echo "Final Performance item/sec : $FPS" +#输出CompileTime +CompileTime=`cat $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|tr -d '\b\r'|grep -Eo "[0-9]*ms/sample"|awk -F "ms/sample" '{print $1}'|awk '{sum+=$1} END {print"",sum/1000}'|awk '{print $1}'` # #输出训练精度,需要模型审视修改 train_accuracy=`grep "test marital AUC" ${cur_path}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|awk '{print $4}'` @@ -162,4 +164,5 @@ echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/output/$ASCEND_DEVICE_ID/$ echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CompileTime = ${CompileTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log diff --git a/TensorFlow/built-in/recommendation/DeepFM_ID0030_for_TensorFlow/test/train_RT2_performance_1p.sh b/TensorFlow/built-in/recommendation/DeepFM_ID0030_for_TensorFlow/test/train_RT2_performance_1p.sh index f860ce5c383c4cdc65c5bb78d751137f309f9056..61cc7d9aa75d4ab84bd7e7bdc6061959e4de39d0 100644 --- a/TensorFlow/built-in/recommendation/DeepFM_ID0030_for_TensorFlow/test/train_RT2_performance_1p.sh +++ b/TensorFlow/built-in/recommendation/DeepFM_ID0030_for_TensorFlow/test/train_RT2_performance_1p.sh @@ -147,6 +147,9 @@ ActualFPS=`grep "fps" $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE temp1=`echo "1000 * ${batch_size}"|bc` TrainingTime=`echo "scale=2;${temp1} / ${ActualFPS}"|bc` +#输出CompileTime +CompileTime=`grep "elapsed" $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log |head -n 1|awk '{print $19}' | awk -F "," '{print $1}'` + ActualLoss=`grep "loss" $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log | awk 'END {print $9}' |tr -d ,` #关键信息打印到${CaseName}.log中,不需要修改 @@ -159,4 +162,5 @@ echo "ActualFPS = ${ActualFPS}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CompileTime = ${CompileTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log sed -i -e '/ModuleNotFoundError/d' $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log