diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_full_1p.sh b/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_full_1p.sh index beb1e3c475e50674378d8bfbe9d311a3defb8cb5..5d93f3a41f9063d29ac0bb157fc1855edc19cd90 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_full_1p.sh +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_full_1p.sh @@ -27,7 +27,7 @@ display_step=10 #维持参数,以下不需要修改 over_dump=False - +precision_mode="allow_mix_precision" # 帮助信息,不需要修改 if [[ $1 == --help || $1 == -h ]];then @@ -104,7 +104,11 @@ cp configs/config.py configs/config.py.run cd $cur_path/../ start=$(date +%s) -nohup python3 train.py --data_path=$data_path --ckpt_path=$cur_path/output/$ASCEND_DEVICE_ID/ckpt > $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log 2>&1 & +nohup python3 train.py --data_path=$data_path \ + --ckpt_path=$cur_path/output/$ASCEND_DEVICE_ID/ckpt \ + --train_size=$train_size \ + --precision_mode=$precision_mode \ + --display_step=$display_step > $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log 2>&1 & wait end=$(date +%s) e2e_time=$(( $end - $start )) diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_full_8p.sh b/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_full_8p.sh index 776939b22cf514533d09956bdb978fc3e2f24133..d74293f30808daee2764c06475006ef13f82125f 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_full_8p.sh +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_full_8p.sh @@ -31,7 +31,7 @@ n_epoches=8 #维持参数,以下不需要修改 over_dump=False - +precision_mode="allow_mix_precision" # 帮助信息,不需要修改 if [[ $1 == --help || $1 == -h ]];then @@ -122,7 +122,11 @@ do else mkdir -p $cur_path/output/${ASCEND_DEVICE_ID} fi - nohup python3 train.py --data_path=$data_path --ckpt_path=$cur_path/output/$ASCEND_DEVICE_ID/ckpt > $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log 2>&1 & + nohup python3 train.py --data_path=$data_path \ + --ckpt_path=$cur_path/output/$ASCEND_DEVICE_ID/ckpt \ + --train_size=$train_size \ + --precision_mode=$precision_mode \ + --display_step=$display_step > $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log 2>&1 & done wait end=$(date +%s) diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_performance_1p.sh b/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_performance_1p.sh index a09fdda8a09bff29fa4c57cde95dc3a58e433f63..506c1fd93e43fc6a8906cb061feb9ded7bf407aa 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_performance_1p.sh +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_performance_1p.sh @@ -28,7 +28,7 @@ display_step=10 #维持参数,以下不需要修改 over_dump=False - +precision_mode="allow_mix_precision" # 帮助信息,不需要修改 if [[ $1 == --help || $1 == -h ]];then echo"usage:./train_performance_1p.sh " @@ -94,10 +94,13 @@ sed -i "s%59761827%${train_size}%p" configs/config.py sed -i "s%display_step = 100%display_step = $display_step%p" configs/config.py #echo `cat configs/config.py |uniq > configs/config.py; cp -f configs/config.py configs/config.py.run` cp configs/config.py configs/config.py.run - #训练执行 start=$(date +%s) -nohup python3 train.py --data_path=$data_path --ckpt_path=$cur_path/output/$ASCEND_DEVICE_ID/ckpt --train_size =$train_size --display_step=$display_step > $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log 2>&1 & +nohup python3 train.py --data_path=$data_path \ + --ckpt_path=$cur_path/output/$ASCEND_DEVICE_ID/ckpt \ + --train_size=$train_size \ + --precision_mode=$precision_mode \ + --display_step=$display_step > $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log 2>&1 & wait end=$(date +%s) e2e_time=$(( $end - $start )) diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_performance_8p.sh b/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_performance_8p.sh index 35585d56032a9b59d5c3ca86b14903894b68056f..abfb0abcdacc712acf5070768b6cd4e595eeae15 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_performance_8p.sh +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/test/train_performance_8p.sh @@ -30,7 +30,7 @@ n_epoches=4 #维持参数,以下不需要修改 over_dump=False - +precision_mode="allow_mix_precision" # 帮助信息,不需要修改 if [[ $1 == --help || $1 == -h ]];then echo"usage:./train_performance_1p.sh " @@ -100,7 +100,11 @@ do #执行训练脚本,以下传参不需要修改,其他需要模型审视修改 - nohup python3 train.py --data_path=$data_path --ckpt_path=$cur_path/output/$ASCEND_DEVICE_ID/ckpt --train_size =$train_size --display_step=$display_step > $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log 2>&1 & + nohup python3 train.py --data_path=$data_path \ + --ckpt_path=$cur_path/output/$ASCEND_DEVICE_ID/ckpt \ + --train_size=$train_size \ + --precision_mode=$precision_mode \ + --display_step=$display_step > $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log 2>&1 & done wait diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/train.py b/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/train.py index 1f7e5fd959f6f3d510f1e55f51ae3a911e949348..0865576db300eb8f5c9b796f826fc37b694b71b0 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/train.py +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2712_for_TensorFlow/train.py @@ -280,6 +280,9 @@ def parse_args(): help="size of train data") parser.add_argument("--display_step", default= config.display_step, help="display step") + parser.add_argument('--precision_mode', default='allow_mix_precision', + help='allow_fp32_to_fp16/force_fp16/ ' + 'must_keep_origin_dtype/allow_mix_precision.') args = parser.parse_args() '''args, unknown_args = parser.parse_known_args() if len(unknown_args) > 0: @@ -291,7 +294,7 @@ def parse_args(): if __name__ == '__main__': display_step = config.display_step - os.environ["TF_ENABLE_AUTO_MIXED_PRECISION_GRAPH_REWRITE"] = "1" + #os.environ["TF_ENABLE_AUTO_MIXED_PRECISION_GRAPH_REWRITE"] = "1" tag = algo Base_path = config.BASE_DIR @@ -336,10 +339,11 @@ if __name__ == '__main__': #custom_op.parameter_map["mix_compile_mode"].b = True #开启混合计算,根据实际情况配置 custom_op.parameter_map["use_off_line"].b = True custom_op.parameter_map["min_group_size"].b = 1 - custom_op.parameter_map["precision_mode"].s = tf.compat.as_bytes("allow_mix_precision") + custom_op.parameter_map["precision_mode"].s = tf.compat.as_bytes(args.precision_mode) custom_op.parameter_map["hcom_parallel"].b = True custom_op.parameter_map["iterations_per_loop"].i = config.iterations_per_loop - custom_op.parameter_map["modify_mixlist"].s = tf.compat.as_bytes("ops_info.json") + if args.precision_mode == "allow_mix_precision": + custom_op.parameter_map["modify_mixlist"].s = tf.compat.as_bytes("ops_info.json") custom_op.parameter_map["fusion_switch_file"].s = tf.compat.as_bytes("fusion_switch.cfg") #aic err debug # custom_op.parameter_map["enable_exception_dump"].i = 1 @@ -536,4 +540,4 @@ if __name__ == '__main__': if mode == 'train' and rank_size > 1: sess.run(npu_shutdown) ############### for hccl $^?################ - sess.close() + sess.close() \ No newline at end of file