diff --git a/TensorFlow/built-in/audio/Tacotron2-v1-Encoder_ID1997_for_TensorFlow/test/train_RT2_performance_1p.sh b/TensorFlow/built-in/audio/Tacotron2-v1-Encoder_ID1997_for_TensorFlow/test/train_RT2_performance_1p.sh index 88fcb9ff1b0fa008374671e7884f0292fd2bddb0..23d5e23c73be818d3351b4eb138aced421a0fd71 100644 --- a/TensorFlow/built-in/audio/Tacotron2-v1-Encoder_ID1997_for_TensorFlow/test/train_RT2_performance_1p.sh +++ b/TensorFlow/built-in/audio/Tacotron2-v1-Encoder_ID1997_for_TensorFlow/test/train_RT2_performance_1p.sh @@ -171,6 +171,9 @@ TrainingTime=`awk 'BEGIN {printf "%.2f\n", '1000'*'${batch_size}'/'${FPS}'}'` #打印,不需要修改 echo "Final Performance images/sec : $FPS" +#输出CompileTime +CompileTime=`grep "perf" $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log |head -n 1|awk '{sum+=$(NF)} END {print sum}'` + #输出训练精度,需要模型审视修改 #train_accuracy=`grep "Accuracy:" $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log | awk -F " " '{print $2}'` #打印,不需要修改 @@ -207,3 +210,4 @@ echo "ActualFPS = ${ActualFPS}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CompileTime = ${CompileTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log diff --git a/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/models/research/object_detection/model_main_rt.py b/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/models/research/object_detection/model_main_rt.py index 7702d48404b3c78ff63f3e3b26fc9d8f307ffa43..dc894b8cb1f360700eee7282b2eb6c229b0fce65 100644 --- a/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/models/research/object_detection/model_main_rt.py +++ b/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/models/research/object_detection/model_main_rt.py @@ -86,6 +86,8 @@ class DLLoggerHook(tf.estimator.SessionRunHook): def after_run(self, run_context, run_values): throughput = (self.global_batch_size / (time.time() - self.t0)) (global_step, lr) = run_values.results + if global_step <= 5: + print("global step: {}, step/s: {}".format(global_step, (time.time() - self.t0))) self.meters['train_throughput'].update(throughput) def end(self, session): @@ -152,10 +154,7 @@ def main(unused_argv): ##################################NPU_modify end################################### eval_hooks = [] for x in range(FLAGS.eval_count): - start = time.time() estimator.train(train_input_fn, hooks=npu_hooks_append(hooks_list=train_hooks), steps=(train_steps // FLAGS.eval_count)) - estimator_time = time.time() - start - print("step/s: {:g}".format(estimator_time)) if (get_npu_rank_id() == 0): eval_input_fn = eval_input_fns[0] #eval阻塞,临时规避 diff --git a/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/test/train_RT2_performance_1p.sh b/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/test/train_RT2_performance_1p.sh index c95fc2a345ff4249504b0fb8cbb461b88605089f..c4ba04f32df239a643fc473c8b152afd3f83d47d 100644 --- a/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/test/train_RT2_performance_1p.sh +++ b/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/test/train_RT2_performance_1p.sh @@ -143,7 +143,7 @@ FPS=`awk 'BEGIN{printf "%.2f\n",'${batch_size}'*'${FPS}'}'` echo "Final Performance images/sec : $FPS" #输出CompileTime -CompileTime=`grep "step/s" $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log |awk '{print$2}'` +CompileTime=`grep "step/s:" $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log |head -n 2|awk '{sum+=$5} END {print sum}'` ################################精度结果处理######################### #精度计算,需要根据网络修改 train_accuracy=`grep Precision $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log|grep Average |awk 'NR==1 {print $13}'` diff --git a/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/test/train_RT2_performance_1p_inceptionv2.sh b/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/test/train_RT2_performance_1p_inceptionv2.sh index b70f401d407481b1c238f4afbd3efe7faf6688ce..16ff19a9110d98e8069555c2a276c9eb9e60ed05 100644 --- a/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/test/train_RT2_performance_1p_inceptionv2.sh +++ b/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/test/train_RT2_performance_1p_inceptionv2.sh @@ -117,6 +117,8 @@ FPS=`grep -a 'INFO:tensorflow:global_step/sec: ' $cur_path/output/${ASCEND_DEVIC FPS=`awk 'BEGIN{printf "%.2f\n",'${batch_size}'*'${RANK_SIZE}'*'${FPS}'}'` echo "Final Performance images/sec : $FPS" +#输出CompileTime +CompileTime=`grep "step/s:" $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log |head -n 2|awk '{sum+=$5} END {print sum}'` ################################E2E训练时长########################## echo "Final Training Duration sec : $e2e_time" @@ -139,4 +141,5 @@ echo "ActualFPS = ${ActualFPS}" >> $cur_path/output/${ASCEND_DEVICE_ID}/${CaseNa echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/${ASCEND_DEVICE_ID}/${CaseName}.log echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/${ASCEND_DEVICE_ID}/${CaseName}.log echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/${ASCEND_DEVICE_ID}/${CaseName}.log +echo "CompileTime = ${CompileTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log diff --git a/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/test/train_RT2_performance_1p_mobilenetv1_fpn.sh b/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/test/train_RT2_performance_1p_mobilenetv1_fpn.sh index a690a02b44a769d6197cd0357d78a5125d452bb7..80589aa30f66a62fff891ec943485f3757693f74 100644 --- a/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/test/train_RT2_performance_1p_mobilenetv1_fpn.sh +++ b/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/test/train_RT2_performance_1p_mobilenetv1_fpn.sh @@ -143,6 +143,9 @@ FPS=`grep -a 'INFO:tensorflow:global_step/sec: ' $cur_path/output/$ASCEND_DEVICE FPS=`awk 'BEGIN{printf "%.2f\n",'${batch_size}'*'${FPS}'}'` echo "Final Performance images/sec : $FPS" + +#输出CompileTime +CompileTime=`grep "step/s:" $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log |head -n 2|awk '{sum+=$5} END {print sum}'` ################################精度结果处理######################### #精度计算,需要根据网络修改 cp -r ${ckpt_path} $cur_path/output/$ASCEND_DEVICE_ID @@ -171,4 +174,5 @@ echo "ActualFPS = ${ActualFPS}" >> $cur_path/output/${ASCEND_DEVICE_ID}/${CaseNa echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/${ASCEND_DEVICE_ID}/${CaseName}.log echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/${ASCEND_DEVICE_ID}/${CaseName}.log echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/${ASCEND_DEVICE_ID}/${CaseName}.log +echo "CompileTime = ${CompileTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log conda deactivate diff --git a/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/test/train_RT2_performance_1p_mobilenetv2.sh b/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/test/train_RT2_performance_1p_mobilenetv2.sh index b7251d32828716906667d7b9f564079d1fe45d50..6f267d910cead8cf016410e6c6bb4faedb8bdec4 100644 --- a/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/test/train_RT2_performance_1p_mobilenetv2.sh +++ b/TensorFlow/built-in/cv/detection/SSD-Resnet50V1-FPN_ID1463_for_TensorFlow/test/train_RT2_performance_1p_mobilenetv2.sh @@ -140,6 +140,8 @@ step_sec=`grep -a 'INFO:tensorflow:global_step/sec: ' $cur_path/output/$ASCEND_D FPS=`awk 'BEGIN{printf "%.2f\n",'${batch_size}'*'${step_sec}'}'` echo "Final Performance images/sec : ${FPS}" +#输出CompileTime +CompileTime=`grep "step/s:" $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log |head -n 2|awk '{sum+=$5} END {print sum}'` #################################精度结果处理######################### # 精度计算,需要根据网络修改 train_accuracy=`grep Precision $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log|awk 'NR==1 {print $13}'` @@ -171,3 +173,4 @@ echo "TrainingTime = ${TrainingTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${Ca echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CompileTime = ${CompileTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log