diff --git a/TensorFlow/contrib/cv/DifferentiableBinarization/test/.keep b/TensorFlow/contrib/cv/DifferentiableBinarization/test/.keep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TensorFlow/contrib/cv/DifferentiableBinarization/test/IMG_0059.JPG b/TensorFlow/contrib/cv/DifferentiableBinarization/test/IMG_0059.JPG deleted file mode 100644 index ac1acf9a41d5fa850572fd131b8ecad9eed3669a..0000000000000000000000000000000000000000 Binary files a/TensorFlow/contrib/cv/DifferentiableBinarization/test/IMG_0059.JPG and /dev/null differ diff --git a/TensorFlow/contrib/cv/DifferentiableBinarization/test/IMG_0080.JPG b/TensorFlow/contrib/cv/DifferentiableBinarization/test/IMG_0080.JPG deleted file mode 100644 index f228aa3774e9f8b689391076e56888b4872df124..0000000000000000000000000000000000000000 Binary files a/TensorFlow/contrib/cv/DifferentiableBinarization/test/IMG_0080.JPG and /dev/null differ diff --git a/TensorFlow/contrib/cv/DifferentiableBinarization/test/img1095.jpg b/TensorFlow/contrib/cv/DifferentiableBinarization/test/img1095.jpg deleted file mode 100644 index c7e4e515f0def411d558c8a5c28a797a7f0cbb23..0000000000000000000000000000000000000000 Binary files a/TensorFlow/contrib/cv/DifferentiableBinarization/test/img1095.jpg and /dev/null differ diff --git a/TensorFlow/contrib/cv/DifferentiableBinarization/test/img192.jpg b/TensorFlow/contrib/cv/DifferentiableBinarization/test/img192.jpg deleted file mode 100644 index dbb678a50681ba8987f8edbecacf9b1da9b6c2ff..0000000000000000000000000000000000000000 Binary files a/TensorFlow/contrib/cv/DifferentiableBinarization/test/img192.jpg and /dev/null differ diff --git a/TensorFlow/contrib/cv/DifferentiableBinarization/test/img795.jpg b/TensorFlow/contrib/cv/DifferentiableBinarization/test/img795.jpg deleted file mode 100644 index 83660e736a41c8aec7c6363da9d46e4b17000989..0000000000000000000000000000000000000000 Binary files a/TensorFlow/contrib/cv/DifferentiableBinarization/test/img795.jpg and /dev/null differ diff --git a/TensorFlow/contrib/cv/DifferentiableBinarization/test/train_full_1p.sh b/TensorFlow/contrib/cv/DifferentiableBinarization/test/train_full_1p.sh new file mode 100644 index 0000000000000000000000000000000000000000..a68051cd2031f0eafa71310b8d40fd46299f1076 --- /dev/null +++ b/TensorFlow/contrib/cv/DifferentiableBinarization/test/train_full_1p.sh @@ -0,0 +1,195 @@ +#!/bin/bash + +########################################################## +#########第3行 至 100行,请一定不要、不要、不要修改########## +#########第3行 至 100行,请一定不要、不要、不要修改########## +#########第3行 至 100行,请一定不要、不要、不要修改########## +########################################################## +# shell脚本所在路径 +cur_path=`echo $(cd $(dirname $0);pwd)` + +# 判断当前shell是否是performance +perf_flag=`echo $0 | grep performance | wc -l` + +# 当前执行网络的名称 +Network=`echo $(cd $(dirname $0);pwd) | awk -F"/" '{print $(NF-1)}'` + +export RANK_SIZE=1 +export RANK_ID=0 +export JOB_ID=10087 + +# 路径参数初始化 +data_path="" +output_path="" + +# 帮助信息,不需要修改 +if [[ $1 == --help || $1 == -h ]];then + echo"usage:./train_performance_1P.sh " + echo " " + echo "parameter explain: + --data_path # dataset of training + --output_path # output of training + --train_steps # max_step for training + --train_epochs # max_epoch for training + --batch_size # batch size + -h/--help show help message + " + exit 1 +fi + +# 参数校验,不需要修改 +for para in $* +do + if [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + elif [[ $para == --output_path* ]];then + output_path=`echo ${para#*=}` + elif [[ $para == --train_steps* ]];then + train_steps=`echo ${para#*=}` + elif [[ $para == --train_epochs* ]];then + train_epochs=`echo ${para#*=}` + elif [[ $para == --batch_size* ]];then + batch_size=`echo ${para#*=}` + fi +done + +# 校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be config" + exit 1 +fi + +# 校验是否传入output_path,不需要修改 +if [[ $output_path == "" ]];then + output_path="./test/output/${ASCEND_DEVICE_ID}" +fi + +# 设置打屏日志文件名,请保留,文件名为${print_log} +print_log="./test/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log" +modelarts_flag=${MODELARTS_MODEL_PATH} +if [ x"${modelarts_flag}" != x ]; +then + echo "running without etp..." + print_log_name=`ls /home/ma-user/modelarts/log/ | grep proc-rank` + print_log="/home/ma-user/modelarts/log/${print_log_name}" +fi +echo "### get your log here : ${print_log}" + +CaseName="" +function get_casename() +{ + if [ x"${perf_flag}" = x1 ]; + then + CaseName=${Network}_bs${batch_size}_${RANK_SIZE}'p'_'perf' + else + CaseName=${Network}_bs${batch_size}_${RANK_SIZE}'p'_'acc' + fi +} + +# 跳转到code目录 +cd ${cur_path}/../ +rm -rf ./test/output/${ASCEND_DEVICE_ID} +mkdir -p ./test/output/${ASCEND_DEVICE_ID} + +sed -i "s#dave/db_48_2.0216_2.5701.h5#${data_path}/DifferentiableBinarization/datasets/total_text/dave/db_48_2.0216_2.5701.h5#g" inference.py +sed -i "s#datasets/total_text/test_images#${data_path}/DifferentiableBinarization/datasets/total_text/test_images#g" inference.py + +# 训练开始时间记录,不需要修改 +start_time=$(date +%s) +########################################################## +#########第3行 至 100行,请一定不要、不要、不要修改########## +#########第3行 至 100行,请一定不要、不要、不要修改########## +#########第3行 至 100行,请一定不要、不要、不要修改########## +########################################################## + +#========================================================= +#========================================================= +#========训练执行命令,需要根据您的网络进行修改============== +#========================================================= +#========================================================= +# 基础参数,需要模型审视修改 +# 您的训练数据集在${data_path}路径下,请直接使用这个变量获取 +# 您的训练输出目录在${output_path}路径下,请直接使用这个变量获取 +# 您的其他基础参数,可以自定义增加,但是batch_size请保留,并且设置正确的值 +batch_size=16 +train_epoch=100 + +if [ x"${modelarts_flag}" != x ]; +then + python3.7 ./train.py \ + --train_generator=${data_path}/DifferentiableBinarization/datasets/total_text \ + --val_generator=${data_path}/DifferentiableBinarization/datasets/total_text \ + --epochs=${train_epoch} \ + --output_path=${output_path} 1>${print_log} 2>&1 +else + python3.7 ./train.py \ + --train_generator=${data_path}/DifferentiableBinarization/datasets/total_text \ + --val_generator=${data_path}/DifferentiableBinarization/datasets/total_text \ + --epochs=${train_epoch} \ + --output_path=${output_path} 1>${print_log} 2>&1 +fi +# 性能相关数据 +StepTime=`grep "200/200" ${print_log} | awk '{print $5}' | tr -d "s/step" | tail -n +2 | awk '{sum+=$1} END {print sum/NR}'` +FPS=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'/'${StepTime}'}'` + +# 精度相关数据计算 +train_accuracy=`grep "200/200" ${print_log} | awk '{print $8}' | awk 'END {print $NF}'` +# 提取所有loss打印信息 +grep "200/200" ${print_log} | awk '{print $8}' > ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt + + +########################################################### +#########后面的所有内容请不要修改########################### +#########后面的所有内容请不要修改########################### +#########后面的所有内容请不要修改########################### +########################################################### + +# 判断本次执行是否正确使用Ascend NPU +use_npu_flag=`grep "The model has been compiled on the Ascend AI processor" ${print_log} | wc -l` +if [ x"${use_npu_flag}" == x0 ]; +then + echo "------------------ ERROR NOTICE START ------------------" + echo "ERROR, your task haven't used Ascend NPU, please check your npu Migration." + echo "------------------ ERROR NOTICE END------------------" +else + echo "------------------ INFO NOTICE START------------------" + echo "INFO, your task have used Ascend NPU, please check your result." + echo "------------------ INFO NOTICE END------------------" +fi + +# 获取最终的casename,请保留,case文件名为${CaseName} +get_casename + +# 重命名loss文件 +if [ -f ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt ]; +then + mv ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt ./test/output/${ASCEND_DEVICE_ID}/${CaseName}_loss.txt +fi + +# 训练端到端耗时 +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + +echo "------------------ Final result ------------------" +# 输出性能FPS/单step耗时/端到端耗时 +echo "Final Performance images/sec : $FPS" +echo "Final Performance sec/step : $StepTime" +echo "E2E Training Duration sec : $e2e_time" + +# 输出训练精度 +echo "Final Train Accuracy : ${train_accuracy}" + +# 最后一个迭代loss值,不需要修改 +ActualLoss=(`awk 'END {print $NF}' $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}_loss.txt`) + +#关键信息打印到${CaseName}.log中,不需要修改 +echo "Network = ${Network}" > $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${batch_size}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = `uname -m`" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualFPS = ${FPS}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${StepTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log \ No newline at end of file diff --git a/TensorFlow/contrib/cv/DifferentiableBinarization/test/train_performance_1p.sh b/TensorFlow/contrib/cv/DifferentiableBinarization/test/train_performance_1p.sh new file mode 100644 index 0000000000000000000000000000000000000000..3061385f554ca27358b24ff45a9ecb4153313a34 --- /dev/null +++ b/TensorFlow/contrib/cv/DifferentiableBinarization/test/train_performance_1p.sh @@ -0,0 +1,195 @@ +#!/bin/bash + +########################################################## +#########第3行 至 100行,请一定不要、不要、不要修改########## +#########第3行 至 100行,请一定不要、不要、不要修改########## +#########第3行 至 100行,请一定不要、不要、不要修改########## +########################################################## +# shell脚本所在路径 +cur_path=`echo $(cd $(dirname $0);pwd)` + +# 判断当前shell是否是performance +perf_flag=`echo $0 | grep performance | wc -l` + +# 当前执行网络的名称 +Network=`echo $(cd $(dirname $0);pwd) | awk -F"/" '{print $(NF-1)}'` + +export RANK_SIZE=1 +export RANK_ID=0 +export JOB_ID=10087 + +# 路径参数初始化 +data_path="" +output_path="" + +# 帮助信息,不需要修改 +if [[ $1 == --help || $1 == -h ]];then + echo"usage:./train_performance_1P.sh " + echo " " + echo "parameter explain: + --data_path # dataset of training + --output_path # output of training + --train_steps # max_step for training + --train_epochs # max_epoch for training + --batch_size # batch size + -h/--help show help message + " + exit 1 +fi + +# 参数校验,不需要修改 +for para in $* +do + if [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + elif [[ $para == --output_path* ]];then + output_path=`echo ${para#*=}` + elif [[ $para == --train_steps* ]];then + train_steps=`echo ${para#*=}` + elif [[ $para == --train_epochs* ]];then + train_epochs=`echo ${para#*=}` + elif [[ $para == --batch_size* ]];then + batch_size=`echo ${para#*=}` + fi +done + +# 校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be config" + exit 1 +fi + +# 校验是否传入output_path,不需要修改 +if [[ $output_path == "" ]];then + output_path="./test/output/${ASCEND_DEVICE_ID}" +fi + +# 设置打屏日志文件名,请保留,文件名为${print_log} +print_log="./test/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log" +modelarts_flag=${MODELARTS_MODEL_PATH} +if [ x"${modelarts_flag}" != x ]; +then + echo "running without etp..." + print_log_name=`ls /home/ma-user/modelarts/log/ | grep proc-rank` + print_log="/home/ma-user/modelarts/log/${print_log_name}" +fi +echo "### get your log here : ${print_log}" + +CaseName="" +function get_casename() +{ + if [ x"${perf_flag}" = x1 ]; + then + CaseName=${Network}_bs${batch_size}_${RANK_SIZE}'p'_'perf' + else + CaseName=${Network}_bs${batch_size}_${RANK_SIZE}'p'_'acc' + fi +} + +# 跳转到code目录 +cd ${cur_path}/../ +rm -rf ./test/output/${ASCEND_DEVICE_ID} +mkdir -p ./test/output/${ASCEND_DEVICE_ID} + +sed -i "s#dave/db_48_2.0216_2.5701.h5#${data_path}/DifferentiableBinarization/datasets/total_text/dave/db_48_2.0216_2.5701.h5#g" inference.py +sed -i "s#datasets/total_text/test_images#${data_path}/DifferentiableBinarization/datasets/total_text/test_images#g" inference.py + +# 训练开始时间记录,不需要修改 +start_time=$(date +%s) +########################################################## +#########第3行 至 100行,请一定不要、不要、不要修改########## +#########第3行 至 100行,请一定不要、不要、不要修改########## +#########第3行 至 100行,请一定不要、不要、不要修改########## +########################################################## + +#========================================================= +#========================================================= +#========训练执行命令,需要根据您的网络进行修改============== +#========================================================= +#========================================================= +# 基础参数,需要模型审视修改 +# 您的训练数据集在${data_path}路径下,请直接使用这个变量获取 +# 您的训练输出目录在${output_path}路径下,请直接使用这个变量获取 +# 您的其他基础参数,可以自定义增加,但是batch_size请保留,并且设置正确的值 +batch_size=16 +train_epoch=3 + +if [ x"${modelarts_flag}" != x ]; +then + python3.7 ./train.py \ + --train_generator=${data_path}/DifferentiableBinarization/datasets/total_text \ + --val_generator=${data_path}/DifferentiableBinarization/datasets/total_text \ + --epochs=${train_epoch} \ + --output_path=${output_path} 1>${print_log} 2>&1 +else + python3.7 ./train.py \ + --train_generator=${data_path}/DifferentiableBinarization/datasets/total_text \ + --val_generator=${data_path}/DifferentiableBinarization/datasets/total_text \ + --epochs=${train_epoch} \ + --output_path=${output_path} 1>${print_log} 2>&1 +fi +# 性能相关数据 +StepTime=`grep "200/200" ${print_log} | awk '{print $5}' | tr -d "s/step" | tail -n +2 | awk '{sum+=$1} END {print sum/NR}'` +FPS=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'/'${StepTime}'}'` + +# 精度相关数据计算 +train_accuracy=`grep "200/200" ${print_log} | awk '{print $8}' | awk 'END {print $NF}'` +# 提取所有loss打印信息 +grep "200/200" ${print_log} | awk '{print $8}' > ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt + + +########################################################### +#########后面的所有内容请不要修改########################### +#########后面的所有内容请不要修改########################### +#########后面的所有内容请不要修改########################### +########################################################### + +# 判断本次执行是否正确使用Ascend NPU +use_npu_flag=`grep "The model has been compiled on the Ascend AI processor" ${print_log} | wc -l` +if [ x"${use_npu_flag}" == x0 ]; +then + echo "------------------ ERROR NOTICE START ------------------" + echo "ERROR, your task haven't used Ascend NPU, please check your npu Migration." + echo "------------------ ERROR NOTICE END------------------" +else + echo "------------------ INFO NOTICE START------------------" + echo "INFO, your task have used Ascend NPU, please check your result." + echo "------------------ INFO NOTICE END------------------" +fi + +# 获取最终的casename,请保留,case文件名为${CaseName} +get_casename + +# 重命名loss文件 +if [ -f ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt ]; +then + mv ./test/output/${ASCEND_DEVICE_ID}/my_output_loss.txt ./test/output/${ASCEND_DEVICE_ID}/${CaseName}_loss.txt +fi + +# 训练端到端耗时 +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + +echo "------------------ Final result ------------------" +# 输出性能FPS/单step耗时/端到端耗时 +echo "Final Performance images/sec : $FPS" +echo "Final Performance sec/step : $StepTime" +echo "E2E Training Duration sec : $e2e_time" + +# 输出训练精度 +echo "Final Train Accuracy : ${train_accuracy}" + +# 最后一个迭代loss值,不需要修改 +ActualLoss=(`awk 'END {print $NF}' $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}_loss.txt`) + +#关键信息打印到${CaseName}.log中,不需要修改 +echo "Network = ${Network}" > $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${batch_size}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = `uname -m`" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualFPS = ${FPS}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${StepTime}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualLoss = ${ActualLoss}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainAccuracy = ${train_accuracy}" >> $cur_path/output/$ASCEND_DEVICE_ID/${CaseName}.log \ No newline at end of file diff --git a/TensorFlow/contrib/cv/DifferentiableBinarization/train.py b/TensorFlow/contrib/cv/DifferentiableBinarization/train.py index 16fd7c8efe2abc3afd21d98ab7b713e9c2ef8650..a6e40f6cd4aa5963bf2a02fe6d80a555caeca637 100644 --- a/TensorFlow/contrib/cv/DifferentiableBinarization/train.py +++ b/TensorFlow/contrib/cv/DifferentiableBinarization/train.py @@ -7,15 +7,25 @@ from keras.utils import get_file import os from generator import generate from model import dbnet -checkpoints_dir = f'checkpoints/{datetime.date.today()}' +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument("--train_generator", default='./datasets/total_text', type=str) +parser.add_argument("--val_generator", default='./datasets/total_text', type=str) +parser.add_argument("--epochs", default=100, type=int) +parser.add_argument("--output_path", default='./output', type=str) + +args = parser.parse_args() +today = datetime.date.today() +checkpoints_dir = f'checkpoints/{today}' batch_size = 16 if not osp.exists(checkpoints_dir): os.makedirs(checkpoints_dir) -train_generator = generate('datasets/total_text', batch_size=batch_size, is_training=True) -val_generator = generate('datasets/total_text', batch_size=batch_size, is_training=False) +train_generator = generate(args.train_generator, batch_size=batch_size, is_training=True) +val_generator = generate(args.val_generator, batch_size=batch_size, is_training=False) model, prediction_model = dbnet() resnet_filename = 'ResNet-50-model.keras.h5' @@ -27,17 +37,16 @@ model.compile(optimizer=optimizers.Adam(lr=1e-3), loss={'db_loss': lambda y_true checkpoint = callbacks.ModelCheckpoint( osp.join(checkpoints_dir, 'db_{epoch:02d}_{loss:.4f}_{val_loss:.4f}.h5'), verbose=1, + mode="min", ) """开始训练""" model.fit_generator( generator=train_generator, steps_per_epoch=200, initial_epoch=0, - epochs=100, + epochs=args.epochs, verbose=1, callbacks=[checkpoint], validation_data=val_generator, validation_steps=19 ) - -