From 0d84b4b39ce020b3122a717c1279798a6f96a60d Mon Sep 17 00:00:00 2001
From: hanxiaoxuan <hanxiaoxuan1@huawei.com>
Date: Thu, 29 Dec 2022 16:08:23 +0800
Subject: [PATCH] add compile time 1229

---
 .../test/train_RT2_performance_1p.sh                          | 4 ++++
 .../test/train_RT2_performance_1p.sh                          | 3 +++
 .../built-in/nlp/CNN-CTC_ID0683_for_TensorFlow/main_rt.py     | 2 +-
 .../test/train_RT2_performance_1p.sh                          | 4 ++++
 .../test/train_RT2_performance_1p.sh                          | 4 ++++
 .../test/train_RT2_performance_1p.sh                          | 3 +++
 .../recommendation/DIEN_ID3065_for_TensorFlow/script/train.py | 2 +-
 .../test/train_RT2_performance_1p.sh                          | 4 ++++
 .../test/train_ID3058_MMoE_performance_1p_RT2.sh              | 3 +++
 .../test/train_RT2_performance_1p.sh                          | 4 ++++
 10 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/TensorFlow/built-in/cv/detection/MaskRcnn_ID0011_for_TensorFlow/test/train_RT2_performance_1p.sh b/TensorFlow/built-in/cv/detection/MaskRcnn_ID0011_for_TensorFlow/test/train_RT2_performance_1p.sh
index 3f0ce9cc4..e2b7d999b 100644
--- a/TensorFlow/built-in/cv/detection/MaskRcnn_ID0011_for_TensorFlow/test/train_RT2_performance_1p.sh
+++ b/TensorFlow/built-in/cv/detection/MaskRcnn_ID0011_for_TensorFlow/test/train_RT2_performance_1p.sh
@@ -187,6 +187,9 @@ FPS=`awk 'BEGIN{printf "%f\n",'${batch_size}'*'${RANK_SIZE}'*'${FPSper}'}'`
 #打印，不需要修改
 echo "Final Performance images/sec : $FPS"
 
+#输出CompileTime
+CompileTime=`grep "INFO:tensorflow:loss" $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log |head -n 2|awk '{if (NR>1) print $7}' | awk -F '(' '{print $2}'`
+
 #输出训练精度,需要模型审视修改
 train_accuracy=`grep "Average Precision" $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|head -1|awk '{print $13}'`
 #打印，不需要修改
@@ -222,3 +225,4 @@ echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${Ca
 #echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
 echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
 echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
+echo "CompileTime = ${CompileTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
diff --git a/TensorFlow/built-in/cv/image_classification/PixelLink_ID3056_for_TensorFlow/test/train_RT2_performance_1p.sh b/TensorFlow/built-in/cv/image_classification/PixelLink_ID3056_for_TensorFlow/test/train_RT2_performance_1p.sh
index e6ec771ef..2ebb42aa7 100644
--- a/TensorFlow/built-in/cv/image_classification/PixelLink_ID3056_for_TensorFlow/test/train_RT2_performance_1p.sh
+++ b/TensorFlow/built-in/cv/image_classification/PixelLink_ID3056_for_TensorFlow/test/train_RT2_performance_1p.sh
@@ -136,6 +136,8 @@ FPS=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'/'${TrainingTime}'}'`
 #打印，不需要修改
 echo "Final Performance item/sec : $FPS"
 
+#输出CompileTime
+CompileTime=`grep "sec/step" $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log |head -n 1|awk '{print $7}' | awk -F '(' '{print $2}'`
 
 # #输出训练精度,需要模型审视修改
 #train_accuracy=`grep "test AUC" ${cur_path}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|awk '{print $3}'`
@@ -169,3 +171,4 @@ echo "ActualFPS = ${ActualFPS}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName
 echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
 echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
 echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
+echo "CompileTime = ${CompileTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
diff --git a/TensorFlow/built-in/nlp/CNN-CTC_ID0683_for_TensorFlow/main_rt.py b/TensorFlow/built-in/nlp/CNN-CTC_ID0683_for_TensorFlow/main_rt.py
index b85b57053..701ac1ddd 100644
--- a/TensorFlow/built-in/nlp/CNN-CTC_ID0683_for_TensorFlow/main_rt.py
+++ b/TensorFlow/built-in/nlp/CNN-CTC_ID0683_for_TensorFlow/main_rt.py
@@ -105,7 +105,7 @@ def train(train_dir=None, val_dir=None, mode='train'):
 
             # the training part
             for cur_batch in range(num_batches_per_epoch):
-                if (cur_batch + 1) % 100 == 0:
+                if (cur_batch + 1) % 100 == 0 or cur_batch <= 5:
                     print('batch', cur_batch, ': time', time.time() - batch_time)
                 batch_time = time.time()
                 indexs = [shuffle_idx[i % num_train_samples] for i in
diff --git a/TensorFlow/built-in/nlp/CNN-CTC_ID0683_for_TensorFlow/test/train_RT2_performance_1p.sh b/TensorFlow/built-in/nlp/CNN-CTC_ID0683_for_TensorFlow/test/train_RT2_performance_1p.sh
index ed46648fd..d613f961f 100644
--- a/TensorFlow/built-in/nlp/CNN-CTC_ID0683_for_TensorFlow/test/train_RT2_performance_1p.sh
+++ b/TensorFlow/built-in/nlp/CNN-CTC_ID0683_for_TensorFlow/test/train_RT2_performance_1p.sh
@@ -69,6 +69,9 @@ FPS=`awk 'BEGIN{printf "%.2f\n",'${batch_size}'/'${TrainingTime}'}'`
 #打印，不需要修改
 echo "Final Performance images/sec : $FPS"
 
+#输出CompileTime
+CompileTime=`grep "time" $cur_path/test/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log | grep batch |head -n 3|awk '{sum+=$5} END {print sum}'`
+
 #输出训练精度,需要模型审视修改
 train_accuracy=`grep "accuracy" $cur_path/test/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log|awk 'END {print $7}'|cut -d , -f 1`
 #打印，不需要修改
@@ -103,3 +106,4 @@ echo "TrainingTime = ${TrainingTime}" >> $cur_path/test/output/$ASCEND_DEVICE_ID
 echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log
 echo "ActualLoss = ${ActualLoss}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log
 echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log
+echo "CompileTime = ${CompileTime}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log
diff --git a/TensorFlow/built-in/nlp/DS-CNN_RT2_ID1769_for_TensorFlow/test/train_RT2_performance_1p.sh b/TensorFlow/built-in/nlp/DS-CNN_RT2_ID1769_for_TensorFlow/test/train_RT2_performance_1p.sh
index 3cc15b418..e7f13c725 100644
--- a/TensorFlow/built-in/nlp/DS-CNN_RT2_ID1769_for_TensorFlow/test/train_RT2_performance_1p.sh
+++ b/TensorFlow/built-in/nlp/DS-CNN_RT2_ID1769_for_TensorFlow/test/train_RT2_performance_1p.sh
@@ -172,6 +172,9 @@ FPS=`awk 'BEGIN{printf "%.2f\n",'${batch_size}'*'${step_per_s}'}'`
 #打印，不需要修改
 echo "Final Performance images/sec : $FPS"
 
+#输出CompileTime
+CompileTime=`grep 'train duration:' $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log |head -n 1|awk '{print $(NF)}'`
+
 #输出训练精度,需要模型审视修改
 train_accuracy=`grep 'INFO:tensorflow.*Validation accuracy' $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log|awk 'END {print $6}'|awk -F % 'END {print $1}'`
 train_accuracy=`awk 'BEGIN{printf "%.4f\n",'${train_accuracy}'/100}'`
@@ -208,3 +211,4 @@ echo "TrainingTime = ${TrainingTime}"     >> $cur_path/output/$ASCEND_DEVICE_ID/
 echo "ActualLoss = ${ActualLoss}"         >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
 echo "E2ETrainingTime = ${e2e_time}"      >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
 echo "TrainAccuracy = ${train_accuracy}"  >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
+echo "CompileTime = ${CompileTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
diff --git a/TensorFlow/built-in/nlp/Siamese_ID0506_for_TensorFlow/test/train_RT2_performance_1p.sh b/TensorFlow/built-in/nlp/Siamese_ID0506_for_TensorFlow/test/train_RT2_performance_1p.sh
index 2a5cb1c2b..7877d6fe9 100644
--- a/TensorFlow/built-in/nlp/Siamese_ID0506_for_TensorFlow/test/train_RT2_performance_1p.sh
+++ b/TensorFlow/built-in/nlp/Siamese_ID0506_for_TensorFlow/test/train_RT2_performance_1p.sh
@@ -87,6 +87,8 @@ train_accuracy=`grep "TRAIN " $cur_path/test/output/$ASCEND_DEVICE_ID/train_$ASC
 #打印，不需要修改
 #echo "Final Train Accuracy : ${train_accuracy}"
 
+#输出CompileTime
+CompileTime=`grep "time(ms)" $cur_path/test/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log |head -n 1|awk '{print $6/1000}'`
 
 #性能看护结果汇总
 #训练用例信息，不需要修改
@@ -117,3 +119,4 @@ echo "TrainingTime = ${TrainingTime}" >> $cur_path/test/output/$ASCEND_DEVICE_ID
 echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log
 echo "ActualLoss = ${ActualLoss}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log
 echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log
+echo "CompileTime = ${CompileTime}" >> $cur_path/test/output/$ASCEND_DEVICE_ID/${CaseName}.log
diff --git a/TensorFlow/built-in/recommendation/DIEN_ID3065_for_TensorFlow/script/train.py b/TensorFlow/built-in/recommendation/DIEN_ID3065_for_TensorFlow/script/train.py
index 8f99a5799..d866126f1 100644
--- a/TensorFlow/built-in/recommendation/DIEN_ID3065_for_TensorFlow/script/train.py
+++ b/TensorFlow/built-in/recommendation/DIEN_ID3065_for_TensorFlow/script/train.py
@@ -233,7 +233,7 @@ def train(
                 aux_loss_sum += aux_loss
                 iter += 1
                 sys.stdout.flush()
-                if (iter % test_iter) == 0:
+                if (iter % test_iter) == 0 or iter <= 5:
                     avg_examples_per_second = batch_size/(end_time - start_time)
                     print("avg_examples_per_second: ", avg_examples_per_second)
                     print('iter: %d ----> train_loss: %.4f ---- train_accuracy: %.4f ---- train_aux_loss: %.4f ---- perf: %.4f' % \
diff --git a/TensorFlow/built-in/recommendation/DIEN_ID3065_for_TensorFlow/test/train_RT2_performance_1p.sh b/TensorFlow/built-in/recommendation/DIEN_ID3065_for_TensorFlow/test/train_RT2_performance_1p.sh
index a397bc8f7..b0ad15296 100644
--- a/TensorFlow/built-in/recommendation/DIEN_ID3065_for_TensorFlow/test/train_RT2_performance_1p.sh
+++ b/TensorFlow/built-in/recommendation/DIEN_ID3065_for_TensorFlow/test/train_RT2_performance_1p.sh
@@ -135,6 +135,9 @@ FPS=`grep  avg_examples_per_second $cur_path/output/$ASCEND_DEVICE_ID/train_$ASC
 #打印，不需要修改
 echo "Final Performance item/sec : $FPS"
 
+#输出CompileTime
+CompileTime=`grep "perf:" $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log |head -n 1|awk '{print $14}'`
+
 #输出训练精度,需要模型审视修改
 train_accuracy=`grep "train_accuracy" $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log | awk -F "train_accuracy:" 'END{print $2}' | awk -F ' ' '{print $1}'|sed s/[[:space:]]//g`
 #打印，不需要修改
@@ -170,4 +173,5 @@ echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/output/$ASCEND_DEVICE_ID/$
 echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
 echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
 echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
+echo "CompileTime = ${CompileTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
 
diff --git a/TensorFlow/built-in/recommendation/DeepCTR_Series_for_TensorFlow/test/train_ID3058_MMoE_performance_1p_RT2.sh b/TensorFlow/built-in/recommendation/DeepCTR_Series_for_TensorFlow/test/train_ID3058_MMoE_performance_1p_RT2.sh
index 0063fcacb..91918048a 100644
--- a/TensorFlow/built-in/recommendation/DeepCTR_Series_for_TensorFlow/test/train_ID3058_MMoE_performance_1p_RT2.sh
+++ b/TensorFlow/built-in/recommendation/DeepCTR_Series_for_TensorFlow/test/train_ID3058_MMoE_performance_1p_RT2.sh
@@ -125,6 +125,8 @@ FPS=`awk 'BEGIN{printf "%.2f\n", 1 /'${Time}'*1000000}'`
 #打印，不需要修改
 echo "Final Performance item/sec : $FPS"
 
+#输出CompileTime
+CompileTime=`cat $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|tr -d '\b\r'|grep -Eo "[0-9]*ms/sample"|awk -F "ms/sample" '{print $1}'|awk '{sum+=$1} END {print"",sum/1000}'|awk '{print $1}'`
 
 # #输出训练精度,需要模型审视修改
 train_accuracy=`grep "test marital AUC" ${cur_path}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|awk '{print $4}'`
@@ -162,4 +164,5 @@ echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/output/$ASCEND_DEVICE_ID/$
 echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
 echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
 echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
+echo "CompileTime = ${CompileTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
 
diff --git a/TensorFlow/built-in/recommendation/DeepFM_ID0030_for_TensorFlow/test/train_RT2_performance_1p.sh b/TensorFlow/built-in/recommendation/DeepFM_ID0030_for_TensorFlow/test/train_RT2_performance_1p.sh
index f860ce5c3..61cc7d9aa 100644
--- a/TensorFlow/built-in/recommendation/DeepFM_ID0030_for_TensorFlow/test/train_RT2_performance_1p.sh
+++ b/TensorFlow/built-in/recommendation/DeepFM_ID0030_for_TensorFlow/test/train_RT2_performance_1p.sh
@@ -147,6 +147,9 @@ ActualFPS=`grep "fps" $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE
 temp1=`echo "1000 * ${batch_size}"|bc`
 TrainingTime=`echo "scale=2;${temp1} / ${ActualFPS}"|bc`
 
+#输出CompileTime
+CompileTime=`grep "elapsed" $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log |head -n 1|awk '{print $19}' | awk -F "," '{print $1}'`
+
 ActualLoss=`grep "loss" $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log | awk 'END {print $9}' |tr -d ,`
 
 #关键信息打印到${CaseName}.log中，不需要修改
@@ -159,4 +162,5 @@ echo "ActualFPS = ${ActualFPS}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName
 echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
 echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
 echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
+echo "CompileTime = ${CompileTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log
 sed -i -e '/ModuleNotFoundError/d' $cur_path/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log
-- 
Gitee