diff --git a/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/env.sh b/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/env.sh deleted file mode 100644 index 560102bd3d368ee110034b173f22ede41938d909..0000000000000000000000000000000000000000 --- a/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/env.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -export install_path=/usr/local/Ascend - -if [ -d ${install_path}/toolkit ]; then - export LD_LIBRARY_PATH=/usr/include/hdf5/lib/:/usr/local/:/usr/local/lib/:/usr/lib/:${install_path}/fwkacllib/lib64/:${install_path}/driver/lib64/common/:${install_path}/driver/lib64/driver/:${install_path}/add-ons:${path_lib}:${LD_LIBRARY_PATH} - export PATH=${install_path}/fwkacllib/ccec_compiler/bin:${install_path}/fwkacllib/bin:$PATH - export PYTHONPATH=${install_path}/fwkacllib/python/site-packages:${install_path}/tfplugin/python/site-packages:${install_path}/toolkit/python/site-packages:$PYTHONPATH - export PYTHONPATH=/usr/local/python3.7.5/lib/python3.7/site-packages:$PYTHONPATH - export ASCEND_OPP_PATH=${install_path}/opp -else - if [ -d ${install_path}/nnae/latest ];then - export LD_LIBRARY_PATH=/usr/local/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/lib/:/usr/lib64/:/usr/lib/:${install_path}/nnae/latest/fwkacllib/lib64/:${install_path}/driver/lib64/common/:${install_path}/driver/lib64/driver/:${install_path}/add-ons/:/usr/lib/aarch64_64-linux-gnu:$LD_LIBRARY_PATH - export PATH=$PATH:${install_path}/nnae/latest/fwkacllib/ccec_compiler/bin/:${install_path}/nnae/latest/toolkit/tools/ide_daemon/bin/ - export ASCEND_OPP_PATH=${install_path}/nnae/latest/opp/ - export OPTION_EXEC_EXTERN_PLUGIN_PATH=${install_path}/nnae/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:${install_path}/nnae/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:${install_path}/nnae/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so - export PYTHONPATH=${install_path}/nnae/latest/fwkacllib/python/site-packages/:${install_path}/nnae/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:${install_path}/nnae/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH - export ASCEND_AICPU_PATH=${install_path}/nnae/latest - else - export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:${install_path}/ascend-toolkit/latest/fwkacllib/lib64/:${install_path}/driver/lib64/common/:${install_path}/driver/lib64/driver/:${install_path}/add-ons/:/usr/lib/aarch64-linux-gnu:$LD_LIBRARY_PATH - export PATH=$PATH:${install_path}/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin/:${install_path}/ascend-toolkit/latest/toolkit/tools/ide_daemon/bin/ - export ASCEND_OPP_PATH=${install_path}/ascend-toolkit/latest/opp/ - export OPTION_EXEC_EXTERN_PLUGIN_PATH=${install_path}/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:${install_path}/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:${install_path}/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so - export PYTHONPATH=${install_path}/ascend-toolkit/latest/fwkacllib/python/site-packages/:${install_path}/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:${install_path}/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH - export ASCEND_AICPU_PATH=${install_path}/ascend-toolkit/latest - fi -fi - -export BMMV2_ENABLE=1 -#将Host日志输出到串口,0-关闭/1-开启 -export ASCEND_SLOG_PRINT_TO_STDOUT=0 -#设置默认日志级别,0-debug/1-info/2-warning/3-error -export ASCEND_GLOBAL_LOG_LEVEL=3 -#设置Host侧Event日志开启标志,0-关闭/1-开启 -export ASCEND_GLOBAL_EVENT_ENABLE=0 -#设置是否开启taskque,0-关闭/1-开启 -export TASK_QUEUE_ENABLE=1 -#设置是否开启PTCopy,0-关闭/1-开启 -export PTCOPY_ENABLE=1 -#设置是否开启combined标志,0-关闭/1-开启 -export COMBINED_ENABLE=1 -#HCCL白名单开关,1-关闭/0-开启 -export HCCL_WHITELIST_DISABLE=1 - -export SCALAR_TO_HOST_MEM=1 - -#设置Device侧日志等级为error -${install_path}/driver/tools/msnpureport -d 0 -g error -${install_path}/driver/tools/msnpureport -d 1 -g error -${install_path}/driver/tools/msnpureport -d 2 -g error -${install_path}/driver/tools/msnpureport -d 3 -g error -${install_path}/driver/tools/msnpureport -d 4 -g error -${install_path}/driver/tools/msnpureport -d 5 -g error -${install_path}/driver/tools/msnpureport -d 6 -g error -${install_path}/driver/tools/msnpureport -d 7 -g error -#关闭Device侧Event日志 -${install_path}/driver/tools/msnpureport -e disable - - -path_lib=$(python3.7 -c """ -import sys -import re -result='' -for index in range(len(sys.path)): - match_sit = re.search('-packages', sys.path[index]) - if match_sit is not None: - match_lib = re.search('lib', sys.path[index]) - - if match_lib is not None: - end=match_lib.span()[1] - result += sys.path[index][0:end] + ':' - - result+=sys.path[index] + '/torch/lib:' -print(result)""" -) - -echo ${path_lib} - -export LD_LIBRARY_PATH=/usr/local/python3.7.5/lib/:${path_lib}:$LD_LIBRARY_PATH diff --git a/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/train_full_1p.sh b/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/train_full_1p.sh index cf7f312b21bd12635f3081d882943ed63fddc2fd..22dcbc5d9172e1838405ce8c1f0a5836784f2565 100644 --- a/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/train_full_1p.sh +++ b/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/train_full_1p.sh @@ -131,7 +131,7 @@ start_time=$(date +%s) check_etp_flag=`env | grep etp_running_flag` etp_flag=`echo ${check_etp_flag#*=}` if [ x"${etp_flag}" != x"true" ];then - source ${test_path_dir}/env.sh + source ${test_path_dir}/env_npu.sh fi #sed -i "s|./data|$data_path|g" examples/cats_and_dogs.py diff --git a/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/train_full_8p.sh b/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/train_full_8p.sh index 5f2c26c1b38b0efbbba84c9e40ed917172ef86ff..ffeea55387da293c8d3171cf234c04c615d7cbe6 100644 --- a/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/train_full_8p.sh +++ b/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/train_full_8p.sh @@ -136,7 +136,7 @@ start_time=$(date +%s) check_etp_flag=`env | grep etp_running_flag` etp_flag=`echo ${check_etp_flag#*=}` if [ x"${etp_flag}" != x"true" ];then - source ${test_path_dir}/env.sh + source ${test_path_dir}/env_npu.sh fi #sed -i "s|./data|$data_path|g" examples/cats_and_dogs.py diff --git a/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/train_performance_1p.sh b/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/train_performance_1p.sh index f70a5af62d9c13dedd4a737454fd55174383a37f..e01677f916b27b0ab96a668f4ab4d195234d7864 100644 --- a/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/train_performance_1p.sh +++ b/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/train_performance_1p.sh @@ -132,7 +132,7 @@ start_time=$(date +%s) check_etp_flag=`env | grep etp_running_flag` etp_flag=`echo ${check_etp_flag#*=}` if [ x"${etp_flag}" != x"true" ];then - source ${test_path_dir}/env.sh + source ${test_path_dir}/env_npu.sh fi #sed -i "s|./data|$data_path|g" examples/cats_and_dogs.py diff --git a/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/train_performance_8p.sh b/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/train_performance_8p.sh index a09ad4b56d5ba6fadb05857c9698a81da1212a25..41aa1a91660aaf980ba4506ef9cb0ad8b3b11715 100644 --- a/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/train_performance_8p.sh +++ b/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/train_performance_8p.sh @@ -136,7 +136,7 @@ start_time=$(date +%s) check_etp_flag=`env | grep etp_running_flag` etp_flag=`echo ${check_etp_flag#*=}` if [ x"${etp_flag}" != x"true" ];then - source ${test_path_dir}/env.sh + source ${test_path_dir}/env_npu.sh fi #sed -i "s|./data|$data_path|g" examples/cats_and_dogs.py diff --git a/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/train_8p.sh b/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/train_8p.sh deleted file mode 100644 index 03a4ffbf1dad64067c4673b718bd0d54413456c1..0000000000000000000000000000000000000000 --- a/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/train_8p.sh +++ /dev/null @@ -1,63 +0,0 @@ -source ./test/env.sh -OUTPATH=data/processed/XLM_en_zh/50k -export NNPU=8 -export WORLD_SIZE=$NNPU -export MASTER_ADDR=$(hostname -I |awk '{print $1}') -export MASTER_PORT=29500 -KERNEL_NUM=$(($(nproc)/8)) -for((RANK_ID=0;RANK_ID