From e2be43e7f8f8f793f5fd37197233c78d2d87d9b0 Mon Sep 17 00:00:00 2001 From: yanqingshang Date: Fri, 4 Dec 2020 15:37:57 +0800 Subject: [PATCH 01/12] new tf to npu --- tools/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tools/.gitkeep diff --git a/tools/.gitkeep b/tools/.gitkeep deleted file mode 100644 index e69de29bb..000000000 -- Gitee From fd681649623707f78c0a62ddfd87aea9cfa15167 Mon Sep 17 00:00:00 2001 From: yanqingshang Date: Fri, 4 Dec 2020 15:43:40 +0800 Subject: [PATCH 02/12] new tf to npu --- conver_tf2npu/ast_impl.py | 116 ++++++++++++++++++++++++++++++++ conver_tf2npu/conver.py | 43 ++++++++++++ conver_tf2npu/conver_by_ast.py | 89 ++++++++++++++++++++++++ conver_tf2npu/file_op.py | 69 +++++++++++++++++++ conver_tf2npu/main.py | 59 ++++++++++++++++ conver_tf2npu/mappings/ast.json | 23 +++++++ conver_tf2npu/util.py | 40 +++++++++++ conver_tf2npu/util_global.py | 33 +++++++++ 8 files changed, 472 insertions(+) create mode 100644 conver_tf2npu/ast_impl.py create mode 100644 conver_tf2npu/conver.py create mode 100644 conver_tf2npu/conver_by_ast.py create mode 100644 conver_tf2npu/file_op.py create mode 100644 conver_tf2npu/main.py create mode 100644 conver_tf2npu/mappings/ast.json create mode 100644 conver_tf2npu/util.py create mode 100644 conver_tf2npu/util_global.py diff --git a/conver_tf2npu/ast_impl.py b/conver_tf2npu/ast_impl.py new file mode 100644 index 000000000..2d3b99ca0 --- /dev/null +++ b/conver_tf2npu/ast_impl.py @@ -0,0 +1,116 @@ +# Copyright 2020 Huawei Technologies Co.,Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import ast +import util_global +from util import log_success_report +from util import log_migration_report + +def attribute(node): + log_success_report(getattr(node, "lineno", "None"), node.attr) + if node.attr == 'dropout': + node.value = ast.Name(id=util_global.get_value(node.attr)[0], ctx=ast.Load()) + else: + node = ast.Name(id=util_global.get_value(node.attr)[0], ctx=ast.Load()) + util_global.set_value('need_conver', True) + return node + +def import_from(node): + if node.module != None: + values = node.module.split(".") + if "keras" in values: + log_migration_report(getattr(node, "lineno", "None"), node.attr) + util_global.set_value('need_conver', True) + +def ast_import(node): + for value in node.names: + if isinstence(value, ast.alias): + values = value.name.split(".") + if "keras" in values: + log_migration_report(getattr(node, "lineno", "None"), node.attr) + util_global.set_value('need_conver', True) + +def ast_function_def(node): + log_success_report(getattr(node, "lineno", "None"), node.name) + node.body = [ast.Return(value=ast.Call( + func=ast.Attribute(value=ast.Name(id=util_global.get_value(node.name)[0], + ctx=ast.Load()), attr='gelu', + ctx=ast.Load()), + args=[ast.Name(id='x', ctx=ast.Load())], + keywords=[]))] + util_global.set_value('need_conver', True) + +def ast_call(node): + if isinstance(node.func, ast.Attribute): + if len(node.args) > 0: + if isinstance(node.args[0], ast.Call): + if isinstance(node.args[0].func, ast.Attribute): + if node.args[0].func.attr == 'BroadcastGlobalVariablesHook': + log_success_report(getattr(node, "lineno", "None"), 'BroadcastGlobalVariablesHook') + node.func = ast.Name(id=util_global.get_value('BroadcastGlobalVariablesHook')[0], ctx=ast.Load()) + node.args = [] + util_global.set_value('need_conver', True) + if isinstance(node.func, ast.Attribute) and node.func.attr == 'shard': + log_success_report(getattr(node, "lineno", "None"), 'shard') + node.args = [ast.Call(func=ast.Name(id='get_rank_size', ctx=ast.Load()), args=[], keywords=[]), + ast.Call(func=ast.Name(id='get_rank_id', ctx=ast.Load()), args=[], keywords=[])] + util_global.set_value('need_conver', True) + if isinstance(node.func, ast.Attribute) and (node.func.attr == 'batch' or node.func.attr == 'map_and_batch'): + exist = False + for keyword in node.keywords: + if keyword.arg == 'drop_remainder': + exist = True + if ((isinstance(keyword.value, ast.NameConstant) and keyword.value.value != True) or + (not isinstance(keyword.value, ast.NameConstant))): + log_success_report(getattr(node, "lineno", "None"), node.func.attr) + keyword.value = ast.NameConstant(value=True) + util_global.set_value('need_conver', True) + if not exist: + log_success_report(getattr(node, "lineno", "None"), node.func.attr) + keyword = ast.keyword(arg='drop_remainder', value=ast.NameConstant(value=True)) + node.keywords.insert(0, keyword) + util_global.set_value('need_conver', True) + if (isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) and + node.func.value.id == 'tf' and node.func.attr == 'device'): + log_success_report(getattr(node, "lineno", "None"), node.func.attr) + node.args = [ast.Str(s='/cpu:0')] + util_global.set_value('need_conver', True) + +def insert_npu_import(r_node): + npu_alias = ast.alias(name='npu_ops', asname=None) + npu_import = ast.ImportFrom(module='npu_bridge.estimator', names=[npu_alias], level=0) + r_node.body.insert(0, npu_import) + npu_alias = ast.alias(name='nup_unary_ops', asname=None) + npu_import = ast.ImportFrom(module='npu_bridge.estimator.npu_unary_ops', names=[npu_alias], level=0) + r_node.body.insert(0, npu_import) + +# Formate printing for locate +def node_tree(node:str): + str2list = list(node.replace(' ', '')) + count = 0 + for i, e in enumerate(str2list): + if e == '(': + count += 1 + str2list[i] = '(\n{}'.format('| ' * count) + elif e == ')': + count -= 1 + str2list[i] = '\n{})'.format('| ' * count) + elif e == ',': + str2list[i] = ',\n{}'.format('| ' * count) + elif e == '[': + count += 1 + str2list[i] = '[\n{}'.format('| ' * count) + elif e == ']': + count -= 1 + str2list[i] = '\n{})'.format('| ' * count) \ No newline at end of file diff --git a/conver_tf2npu/conver.py b/conver_tf2npu/conver.py new file mode 100644 index 000000000..efc871916 --- /dev/null +++ b/conver_tf2npu/conver.py @@ -0,0 +1,43 @@ +# Copyright 2020 Huawei Technologies Co.,Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import os +import util_global +from conver_by_ast import conver_ast +from file_op import mkdir +from file_op import mkdir_and_copyfile +from file_op import write_report_terminator +from file_op import abs_join + +def conver(): + print("Begin conver, input file: " + util_global.get_value('input')) + out_path = util_global.get_value('output') + dst_path = os.path.split(util_global.get_value('input').rstrip('\\/'))[-1] + conver_path = os.walk(util_global.get_value('input')) + for path,dir_list,file_list in conver_path: + for file_name in file_list: + out_path_dst = abs_join(dst_path, path.split(dst_path)[1]) + if file_name.endswith(".py"): + util_global.set_value('path', os.path.join(path, file_name)) + mkdir(os.path.join(out_path, out_path_dst)) + conver_ast(path, out_path_dst, file_name) + if util_global.get_value('need_conver', False): + content = "Finish conver file: " + os.path.join(path, file_name) + print(content) + write_report_terminator(content) + else: + mkdir_and_copyfile(path, abs_join(out_path, out_path_dst), file_name) + else: + mkdir_and_copyfile(path, abs_join(out_path, out_path_dst), file_name) + print("Finish conver, output file: " + out_path + "; report file:" + util_global.get_value('report')) diff --git a/conver_tf2npu/conver_by_ast.py b/conver_tf2npu/conver_by_ast.py new file mode 100644 index 000000000..b9dabe303 --- /dev/null +++ b/conver_tf2npu/conver_by_ast.py @@ -0,0 +1,89 @@ +# Copyright 2020 Huawei Technologies Co.,Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import os +import sys +import ast +import astunparse +import util_global +from file_op import write_output_after_conver +from file_op import write_report_after_conver +from util import log_success_report +from ast_impl import attribute +from ast_impl import node_tree +from ast_impl import insert_npu_import +from ast_impl import import_from +from ast_impl import ast_import +from ast_impl import ast_function_def +from ast_impl import ast_call + +class ConverByAst(ast.NodeTransformer): + def generic_visit(self, node): + ast.NodeTransformer.generic_visit(node) + return node + def visit_Attribute(self, node): + if node.attr in util_global.get_value('nn') and isinstance(node.value, ast.Attribute): + if node.value.attr == 'nn': + return attribute(node) + if node.attr in util_global.get_value('estimator') and isinstance(node.value, ast.Attribute): + if node.value.attr == 'estimator': + return attribute(node) + if node.attr in util_global.get_value('hvd'): + if isinstance(node.value, ast.Name): + if 'hvd' in str(node.value.id): + return attribute(node) + if isinstance(node.value, ast.Attribute): + if 'hvd' in str(node.value.id): + return attribute(node) + return node + + def visit_FunctionDef(self, node): + if node.name == 'gelu': + return ast_function_def(node) + self.generic_visit(node) + return node + + def visit_Call(self, node): + node = ast_call(node) + self.generic_visit(node) + return node + + def visit_ImportFrom(self, node): + import_from(node) + self.generic_visit(node) + return node + + def visit_Import(self, node): + ast_import(node) + self.generic_visit(node) + return node + +def conver_ast(path, out_path_dst, file_name): + util_global.set_value('need_conver', False) + file = open(os.path.join(path, file_name), "r") + source = file.read() + r_node = ast.parse(source) + + sys.setrecursionlimit(10000) + visitor = ConverByAst() + visitor.visit(r_node) + ast.fix_missing_locations(r_node) + + if util_global.get_value('need_conver', False): + insert_npu_import(r_node) + dst_content = astunparse.unparse(r_node) + write_output_after_conver(os.path.join(util_global.get_value('output'), out_path_dst, file_name), dst_content) + + if file_name.endswith("a.py"): + write_report_after_conver("only_for_test", file_name, node_tree(ast.dump(r_node))) \ No newline at end of file diff --git a/conver_tf2npu/file_op.py b/conver_tf2npu/file_op.py new file mode 100644 index 000000000..d5045cfe6 --- /dev/null +++ b/conver_tf2npu/file_op.py @@ -0,0 +1,69 @@ +# Copyright 2020 Huawei Technologies Co.,Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import os +import shutil +import util_global + +def befor_clear(): + exit_folder = os.path.exists(util_global.get_value('output')) + if exit_folder: + shutil.rmtree(util_global.get_value('output')) + exit_folder = os.path.exists(util_global.get_value('report')) + if exit_folder: + shutil.rmtree(util_global.get_value('report')) + +def mkdir(path): + folder = os.path.exists(path) + if not folder: + os.makedirs(path) + +def madir_and_copyfile(srcfile, dstpath, file_name): + mkdir(dstpath) + shutil.copyfile(os.path.join(srcfile, file_name), os.path.join(dstpath, file_name)) + +def write_output_after_conver(out_file, dst_content): + file = open(out_file, 'w') + file.write(dst_content) + file.close + +def write_output_after_conver(new_file_path, report_file, dst_content): + mkdir(new_file_path) + file = open(os.path.join(new_file_path, report_file), 'w') + file.write(dst_content) + file.close + +def write_report_terminator(content): + report_path = util_global.get_value('report') + for file in util_global.get_value('report_file'): + if os.path.exists(os.path.join(report_path, file)): + file = open(os.path.join(report_path, file), 'a') + file.write(content) + file.write("\r\n") + file.write("\r\n") + file.close + +def write_conver_report(content, file): + report_path = util_global.get_value('report') + mkdir(report_path) + file = open(os.path.join(report_path, file), 'a') + file.write(content) + file.write("\r\n") + file.close + +def abs_jion(abs1, abs2): + abs2 = os.fspath(abs2) + abs2 = os.path.splitdrive(abs2)[1] + abs2 = abs2.strip('\\/') or abs2 + return os.path.join(abs1, abs2) \ No newline at end of file diff --git a/conver_tf2npu/main.py b/conver_tf2npu/main.py new file mode 100644 index 000000000..13bdc1952 --- /dev/null +++ b/conver_tf2npu/main.py @@ -0,0 +1,59 @@ +# Copyright 2020 Huawei Technologies Co.,Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import sys +import getopt +import util_global +from file_op import before_clear +from conver import conver + +def para_check_and_set(argv): + input = "input" + output = "output" + report = "report" + + try: + opts, args = getopt.getopt(argv, "hi:o:r", ["help", "input=", "output=", "report="]) + except getopt.GetoptError: + print("Parameter error, please check.") + print(' main.py -i -o -r ') + print('or: main.py -input -output -report ') + print('-i or --input: The source script to be converted, Default value: input/') + print('-o or --output: The destination script after converted, Default value: output/') + print('-r or --report: Conversion report, Default value: report/') + sys.exit(2) + + for opt, arg in opts: + if opt in ("-h", "--help"): + print(' main.py -i -o -r ') + print('or: main.py -input -output -report ') + print('-i or --input: The source script to be converted, Default value: input/') + print('-o or --output: The destination script after converted, Default value: output/') + print('-r or --report: Conversion report, Default value: report/') + sys.exit() + elif opt in ("-i", "--input"): + input = arg + elif opt in ("-o", "--output"): + output = arg + elif opt in ("-r", "--report"): + report = arg + util_global.set_value("input", input) + util_global.set_value("output", output) + util_global.set_value("report", report) + +if __name__ == "__main__": + util_global._init() + para_check_and_set(sys.argv[1:]) + before_clear() + conver() \ No newline at end of file diff --git a/conver_tf2npu/mappings/ast.json b/conver_tf2npu/mappings/ast.json new file mode 100644 index 000000000..1a25d82ec --- /dev/null +++ b/conver_tf2npu/mappings/ast.json @@ -0,0 +1,23 @@ +{ + "need_conver": false, + "gelu": ["npu_unary_ops", "tf.gelu", "npu_unary_ops.gelu"], + "dropout": ["npu_ops", "tf.nn.dropout", "npu_ops.dropout"], + "init": ["print", "hvd.init", "None"], + "DistributedOptimizer": ["NPUDistributedOptimizer", "hvd.DistributedOptimizer", "NPUDistributedOptimizer"], + "rank": ["get_rank_id", "hvd.rank", "get_rank_id"], + "local_rank": ["get_local_rank_id", "hvd.local_rank", "get_local_rank_id"], + "size": ["get_rank_size", "hvd.size", "get_rank_size"], + "BroadcastGlobalVariablesHook": ["print", "BroadcastGlobalVariablesHook", "None"], + "shard": ["", "dataset.shard(xxx, xxx)", "dataset.shard(get_rank_size(), get_rank_id())"], + "EstimatorSpec": ["NPUEstimatorSpec", "tf.estimator.EstimatorSpec", "NPUEstimatorSpec"], + "RunConfig": ["NPURunConfig", "tf.estimator.RunConfig", "NPURunConfig"], + "Estimator": ["NPUEstimator", "tf.estimator.Estimator", "NPUEstimator"], + "batch": ["", "batch(xxx)", "batch(xxx, drop_remainder=True)"], + "map_and_batch": ["", "map_and_batch(xxx)", "map_and_batch(xxx, drop_remainder=True)"], + "device": ["", "tf.device(xxx)", "tf.device(/cpu:0)"], + "hvd": ["init", "rank", "local_rank", "size", "DistributedOptimizer"], + "estimator": ["Estimator", "RunConfig", "EstimatorSpec"], + "nn": ["dropout"], + "keras": ["https://support.huaweicloud.com/mprtg-A800_9000_9010/atlasprtg_13_0008.html"], + "report_file": ["success_report.txt", "failed_report.txt", "need_migration_doc.txt"] +} \ No newline at end of file diff --git a/conver_tf2npu/util.py b/conver_tf2npu/util.py new file mode 100644 index 000000000..4dd8ef2d4 --- /dev/null +++ b/conver_tf2npu/util.py @@ -0,0 +1,40 @@ +# Copyright 2020 Huawei Technologies Co.,Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import util_global +from file_op import write_conver_report + +def log_info(msg): + print(util_global.get_value('path') + ': ' + msg) + +def log_info(lineno, msg, file): + content = (util_globel.get_value('path', '') + ':' + str(lineno) + + ' change ' + util_global.get_value(msg)[1] + + ' to ' + util_global.get_value(msg)[2]) + print(content) + write_conver_report(content, file) + +def log_success_report(lineno, msg): + content = (util_globel.get_value('path', '') + ':' + str(lineno) + + ' change ' + util_global.get_value(msg)[1] + + ' to ' + util_global.get_value(msg)[2]) + print(content) + write_conver_report(content, util_global.get_value('reprot_file')[0]) + +def log_migration_report(lineno, msg): + content = (util_globel.get_value('path', '') + ':' + str(lineno) + ' "' + msg + + '" feature needs to be migrated manually, Please refer to the migration guide: ' + + util_global.get_value(msg)[0]) + print(content) + write_conver_report(content, util_global.get_value('reprot_file')[2]) \ No newline at end of file diff --git a/conver_tf2npu/util_global.py b/conver_tf2npu/util_global.py new file mode 100644 index 000000000..e9c9c31ec --- /dev/null +++ b/conver_tf2npu/util_global.py @@ -0,0 +1,33 @@ +# Copyright 2020 Huawei Technologies Co.,Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import json + +def _init(): + global _global_dict + _global_dict = {} + with open('mappings/ast.json', encoding='utf-8') as f: + load_dict = json.load(f) + items = load_dict.items() + for key, value in items: + set_value(key, value) + +def set_value(key, value): + _global_dict[key] = value + +def get_value(key, def_value = None): + try: + return _global_dict[key] + except KeyError: + return def_value -- Gitee From 8837c5664f05262bd2c0e252754a7e519a251e90 Mon Sep 17 00:00:00 2001 From: yanqingshang Date: Fri, 4 Dec 2020 15:57:35 +0800 Subject: [PATCH 03/12] new tf to npu --- .idea/.gitignore | 8 ++++ .idea/misc.xml | 4 ++ .idea/modules.xml | 8 ++++ .idea/tensorflow.iml | 2 + .idea/vcs.xml | 6 +++ cmake-build-debug/CMakeFiles/clion-log.txt | 1 + conver_tf2npu/ast_impl.py | 18 +++++---- conver_tf2npu/conver.py | 1 + conver_tf2npu/conver_by_ast.py | 4 +- conver_tf2npu/file_op.py | 16 ++++---- conver_tf2npu/main.py | 14 +++---- conver_tf2npu/mappings/ast.json | 45 ++++++++++++---------- conver_tf2npu/util.py | 12 +++--- 13 files changed, 87 insertions(+), 52 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/tensorflow.iml create mode 100644 .idea/vcs.xml create mode 100644 cmake-build-debug/CMakeFiles/clion-log.txt diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 000000000..5a1d6ab72 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Datasource local storage ignored files +/../../../../../:\00gitee\00tfadapter\tensorflow\.idea/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 000000000..79b3c9483 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 000000000..2bda9d8e1 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/tensorflow.iml b/.idea/tensorflow.iml new file mode 100644 index 000000000..f08604bb6 --- /dev/null +++ b/.idea/tensorflow.iml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 000000000..94a25f7f4 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/cmake-build-debug/CMakeFiles/clion-log.txt b/cmake-build-debug/CMakeFiles/clion-log.txt new file mode 100644 index 000000000..c7ba5e8f9 --- /dev/null +++ b/cmake-build-debug/CMakeFiles/clion-log.txt @@ -0,0 +1 @@ +Toolchains are not configured Configure diff --git a/conver_tf2npu/ast_impl.py b/conver_tf2npu/ast_impl.py index 2d3b99ca0..bc06bb57a 100644 --- a/conver_tf2npu/ast_impl.py +++ b/conver_tf2npu/ast_impl.py @@ -30,26 +30,28 @@ def import_from(node): if node.module != None: values = node.module.split(".") if "keras" in values: - log_migration_report(getattr(node, "lineno", "None"), node.attr) + log_migration_report(getattr(node, "lineno", "None"), "keras") util_global.set_value('need_conver', True) def ast_import(node): for value in node.names: - if isinstence(value, ast.alias): + if isinstance(value, ast.alias): values = value.name.split(".") if "keras" in values: - log_migration_report(getattr(node, "lineno", "None"), node.attr) + log_migration_report(getattr(node, "lineno", "None"), "keras") util_global.set_value('need_conver', True) def ast_function_def(node): log_success_report(getattr(node, "lineno", "None"), node.name) node.body = [ast.Return(value=ast.Call( func=ast.Attribute(value=ast.Name(id=util_global.get_value(node.name)[0], - ctx=ast.Load()), attr='gelu', - ctx=ast.Load()), + ctx=ast.Load()), attr='gelu', + ctx=ast.Load()), args=[ast.Name(id='x', ctx=ast.Load())], keywords=[]))] + util_global.set_value('need_conver', True) + return node def ast_call(node): if isinstance(node.func, ast.Attribute): @@ -72,7 +74,7 @@ def ast_call(node): if keyword.arg == 'drop_remainder': exist = True if ((isinstance(keyword.value, ast.NameConstant) and keyword.value.value != True) or - (not isinstance(keyword.value, ast.NameConstant))): + (not isinstance(keyword.value, ast.NameConstant))): log_success_report(getattr(node, "lineno", "None"), node.func.attr) keyword.value = ast.NameConstant(value=True) util_global.set_value('need_conver', True) @@ -82,7 +84,7 @@ def ast_call(node): node.keywords.insert(0, keyword) util_global.set_value('need_conver', True) if (isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) and - node.func.value.id == 'tf' and node.func.attr == 'device'): + node.func.value.id == 'tf' and node.func.attr == 'device'): log_success_report(getattr(node, "lineno", "None"), node.func.attr) node.args = [ast.Str(s='/cpu:0')] util_global.set_value('need_conver', True) @@ -91,7 +93,7 @@ def insert_npu_import(r_node): npu_alias = ast.alias(name='npu_ops', asname=None) npu_import = ast.ImportFrom(module='npu_bridge.estimator', names=[npu_alias], level=0) r_node.body.insert(0, npu_import) - npu_alias = ast.alias(name='nup_unary_ops', asname=None) + npu_alias = ast.alias(name='npu_unary_ops', asname=None) npu_import = ast.ImportFrom(module='npu_bridge.estimator.npu_unary_ops', names=[npu_alias], level=0) r_node.body.insert(0, npu_import) diff --git a/conver_tf2npu/conver.py b/conver_tf2npu/conver.py index efc871916..165fcad48 100644 --- a/conver_tf2npu/conver.py +++ b/conver_tf2npu/conver.py @@ -40,4 +40,5 @@ def conver(): mkdir_and_copyfile(path, abs_join(out_path, out_path_dst), file_name) else: mkdir_and_copyfile(path, abs_join(out_path, out_path_dst), file_name) + print("Finish conver, output file: " + out_path + "; report file:" + util_global.get_value('report')) diff --git a/conver_tf2npu/conver_by_ast.py b/conver_tf2npu/conver_by_ast.py index b9dabe303..7efaa30b8 100644 --- a/conver_tf2npu/conver_by_ast.py +++ b/conver_tf2npu/conver_by_ast.py @@ -30,7 +30,7 @@ from ast_impl import ast_call class ConverByAst(ast.NodeTransformer): def generic_visit(self, node): - ast.NodeTransformer.generic_visit(node) + ast.NodeTransformer.generic_visit(self, node) return node def visit_Attribute(self, node): if node.attr in util_global.get_value('nn') and isinstance(node.value, ast.Attribute): @@ -44,7 +44,7 @@ class ConverByAst(ast.NodeTransformer): if 'hvd' in str(node.value.id): return attribute(node) if isinstance(node.value, ast.Attribute): - if 'hvd' in str(node.value.id): + if 'hvd' in str(node.value.attr): return attribute(node) return node diff --git a/conver_tf2npu/file_op.py b/conver_tf2npu/file_op.py index d5045cfe6..5cbc1c124 100644 --- a/conver_tf2npu/file_op.py +++ b/conver_tf2npu/file_op.py @@ -16,7 +16,7 @@ import os import shutil import util_global -def befor_clear(): +def before_clear(): exit_folder = os.path.exists(util_global.get_value('output')) if exit_folder: shutil.rmtree(util_global.get_value('output')) @@ -29,20 +29,20 @@ def mkdir(path): if not folder: os.makedirs(path) -def madir_and_copyfile(srcfile, dstpath, file_name): +def mkdir_and_copyfile(srcfile, dstpath, file_name): mkdir(dstpath) shutil.copyfile(os.path.join(srcfile, file_name), os.path.join(dstpath, file_name)) def write_output_after_conver(out_file, dst_content): file = open(out_file, 'w') file.write(dst_content) - file.close + file.close() -def write_output_after_conver(new_file_path, report_file, dst_content): +def write_report_after_conver(new_file_path, report_file, dst_content): mkdir(new_file_path) file = open(os.path.join(new_file_path, report_file), 'w') file.write(dst_content) - file.close + file.close() def write_report_terminator(content): report_path = util_global.get_value('report') @@ -52,7 +52,7 @@ def write_report_terminator(content): file.write(content) file.write("\r\n") file.write("\r\n") - file.close + file.close() def write_conver_report(content, file): report_path = util_global.get_value('report') @@ -60,9 +60,9 @@ def write_conver_report(content, file): file = open(os.path.join(report_path, file), 'a') file.write(content) file.write("\r\n") - file.close + file.close() -def abs_jion(abs1, abs2): +def abs_join(abs1, abs2): abs2 = os.fspath(abs2) abs2 = os.path.splitdrive(abs2)[1] abs2 = abs2.strip('\\/') or abs2 diff --git a/conver_tf2npu/main.py b/conver_tf2npu/main.py index 13bdc1952..814f8666d 100644 --- a/conver_tf2npu/main.py +++ b/conver_tf2npu/main.py @@ -24,11 +24,11 @@ def para_check_and_set(argv): report = "report" try: - opts, args = getopt.getopt(argv, "hi:o:r", ["help", "input=", "output=", "report="]) + opts, args = getopt.getopt(argv, "hi:o:r:", ["help", "input=", "output=", "report="]) except getopt.GetoptError: - print("Parameter error, please check.") + print('Parameter error, please check.') print(' main.py -i -o -r ') - print('or: main.py -input -output -report ') + print('or: main.py --input= --output= --report=') print('-i or --input: The source script to be converted, Default value: input/') print('-o or --output: The destination script after converted, Default value: output/') print('-r or --report: Conversion report, Default value: report/') @@ -37,7 +37,7 @@ def para_check_and_set(argv): for opt, arg in opts: if opt in ("-h", "--help"): print(' main.py -i -o -r ') - print('or: main.py -input -output -report ') + print('or: main.py --input= --output= --report=') print('-i or --input: The source script to be converted, Default value: input/') print('-o or --output: The destination script after converted, Default value: output/') print('-r or --report: Conversion report, Default value: report/') @@ -48,9 +48,9 @@ def para_check_and_set(argv): output = arg elif opt in ("-r", "--report"): report = arg - util_global.set_value("input", input) - util_global.set_value("output", output) - util_global.set_value("report", report) + util_global.set_value('input', input) + util_global.set_value('output', output) + util_global.set_value('report', report) if __name__ == "__main__": util_global._init() diff --git a/conver_tf2npu/mappings/ast.json b/conver_tf2npu/mappings/ast.json index 1a25d82ec..a4e793360 100644 --- a/conver_tf2npu/mappings/ast.json +++ b/conver_tf2npu/mappings/ast.json @@ -1,23 +1,26 @@ { - "need_conver": false, - "gelu": ["npu_unary_ops", "tf.gelu", "npu_unary_ops.gelu"], - "dropout": ["npu_ops", "tf.nn.dropout", "npu_ops.dropout"], - "init": ["print", "hvd.init", "None"], - "DistributedOptimizer": ["NPUDistributedOptimizer", "hvd.DistributedOptimizer", "NPUDistributedOptimizer"], - "rank": ["get_rank_id", "hvd.rank", "get_rank_id"], - "local_rank": ["get_local_rank_id", "hvd.local_rank", "get_local_rank_id"], - "size": ["get_rank_size", "hvd.size", "get_rank_size"], - "BroadcastGlobalVariablesHook": ["print", "BroadcastGlobalVariablesHook", "None"], - "shard": ["", "dataset.shard(xxx, xxx)", "dataset.shard(get_rank_size(), get_rank_id())"], - "EstimatorSpec": ["NPUEstimatorSpec", "tf.estimator.EstimatorSpec", "NPUEstimatorSpec"], - "RunConfig": ["NPURunConfig", "tf.estimator.RunConfig", "NPURunConfig"], - "Estimator": ["NPUEstimator", "tf.estimator.Estimator", "NPUEstimator"], - "batch": ["", "batch(xxx)", "batch(xxx, drop_remainder=True)"], - "map_and_batch": ["", "map_and_batch(xxx)", "map_and_batch(xxx, drop_remainder=True)"], - "device": ["", "tf.device(xxx)", "tf.device(/cpu:0)"], - "hvd": ["init", "rank", "local_rank", "size", "DistributedOptimizer"], - "estimator": ["Estimator", "RunConfig", "EstimatorSpec"], - "nn": ["dropout"], - "keras": ["https://support.huaweicloud.com/mprtg-A800_9000_9010/atlasprtg_13_0008.html"], - "report_file": ["success_report.txt", "failed_report.txt", "need_migration_doc.txt"] +"need_conver": false, +"gelu": ["npu_unary_ops", "tf.gelu", "npu_unary_ops.gelu"], +"dropout": ["npu_ops", "tf.nn.dropout", "npu_ops.dropout"], +"init": ["print", "hvd.init", "None"], +"DistributedOptimizer": ["NPUDistributedOptimizer", "hvd.DistributedOptimizer", "NPUDistributedOptimizer"], +"rank": ["get_rank_id", "hvd.rank", "get_rank_id"], +"local_rank": ["get_local_rank_id", "hvd.local_rank", "get_local_rank_id"], +"size": ["get_rank_size", "hvd.size", "get_rank_size"], +"BroadcastGlobalVariablesHook": ["print", "BroadcastGlobalVariablesHook", "None"], +"shard": ["", "dataset.shard(xxx, xxx)", "dataset.shard(get_rank_size(), get_rank_id())"], +"EstimatorSpec": ["NPUEstimatorSpec", "tf.estimator.EstimatorSpec", "NPUEstimatorSpec"], +"RunConfig": ["NPURunConfig", "tf.estimator.RunConfig", "NPURunConfig"], +"Estimator": ["NPUEstimator", "tf.estimator.Estimator", "NPUEstimator"], + +"batch": ["", "batch(xxx)", "batch(xxx, drop_remainder=True)"], +"map_and_batch": ["", "map_and_batch(xxx)", "map_and_batch(xxx, drop_remainder=True)"], +"device": ["", "tf.device(xxx)", "tf.device(/cpu:0)"], + +"hvd": ["init", "rank", "local_rank", "size", "DistributedOptimizer"], +"estimator": ["Estimator", "RunConfig", "EstimatorSpec"], +"nn": ["dropout"], +"keras": ["https://support.huaweicloud.com/mprtg-A800_9000_9010/atlasprtg_13_0008.html"], + +"report_file": ["success_report.txt", "failed_report.txt", "need_migration_doc.txt"] } \ No newline at end of file diff --git a/conver_tf2npu/util.py b/conver_tf2npu/util.py index 4dd8ef2d4..06879487a 100644 --- a/conver_tf2npu/util.py +++ b/conver_tf2npu/util.py @@ -19,22 +19,22 @@ def log_info(msg): print(util_global.get_value('path') + ': ' + msg) def log_info(lineno, msg, file): - content = (util_globel.get_value('path', '') + ':' + str(lineno) + + content = (util_global.get_value('path', '') + ':' + str(lineno) + ' change ' + util_global.get_value(msg)[1] + ' to ' + util_global.get_value(msg)[2]) print(content) write_conver_report(content, file) def log_success_report(lineno, msg): - content = (util_globel.get_value('path', '') + ':' + str(lineno) + - ' change ' + util_global.get_value(msg)[1] + + content = (util_global.get_value('path', '') + ':' + str(lineno) + + ' "change ' + util_global.get_value(msg)[1] + ' to ' + util_global.get_value(msg)[2]) print(content) - write_conver_report(content, util_global.get_value('reprot_file')[0]) + write_conver_report(content, util_global.get_value('report_file')[0]) def log_migration_report(lineno, msg): - content = (util_globel.get_value('path', '') + ':' + str(lineno) + ' "' + msg + + content = (util_global.get_value('path', '') + ':' + str(lineno) + ' "' + msg + '" feature needs to be migrated manually, Please refer to the migration guide: ' + util_global.get_value(msg)[0]) print(content) - write_conver_report(content, util_global.get_value('reprot_file')[2]) \ No newline at end of file + write_conver_report(content, util_global.get_value('report_file')[2]) \ No newline at end of file -- Gitee From 713446adc1278cee8ce4ae7afc0eae43b2459b14 Mon Sep 17 00:00:00 2001 From: yanqingshang Date: Fri, 4 Dec 2020 16:13:25 +0800 Subject: [PATCH 04/12] new tf to npu --- conver_tf2npu/ast_impl.py | 8 ++++--- conver_tf2npu/conver.py | 4 ++-- conver_tf2npu/conver_by_ast.py | 4 ++-- conver_tf2npu/file_op.py | 4 ++-- conver_tf2npu/main.py | 10 ++++----- conver_tf2npu/mappings/ast.json | 40 ++++++++++++++++----------------- conver_tf2npu/util.py | 4 ++-- conver_tf2npu/util_global.py | 4 ++-- 8 files changed, 40 insertions(+), 38 deletions(-) diff --git a/conver_tf2npu/ast_impl.py b/conver_tf2npu/ast_impl.py index bc06bb57a..f341d8f01 100644 --- a/conver_tf2npu/ast_impl.py +++ b/conver_tf2npu/ast_impl.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co.,Ltd +# Copyright 2020 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -6,7 +6,7 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software +# Unless REQUIRED by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and @@ -88,6 +88,7 @@ def ast_call(node): log_success_report(getattr(node, "lineno", "None"), node.func.attr) node.args = [ast.Str(s='/cpu:0')] util_global.set_value('need_conver', True) + return node def insert_npu_import(r_node): npu_alias = ast.alias(name='npu_ops', asname=None) @@ -115,4 +116,5 @@ def node_tree(node:str): str2list[i] = '[\n{}'.format('| ' * count) elif e == ']': count -= 1 - str2list[i] = '\n{})'.format('| ' * count) \ No newline at end of file + str2list[i] = '\n{}]'.format('| ' * count) + return ''.join(str2list) \ No newline at end of file diff --git a/conver_tf2npu/conver.py b/conver_tf2npu/conver.py index 165fcad48..5ed22b424 100644 --- a/conver_tf2npu/conver.py +++ b/conver_tf2npu/conver.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co.,Ltd +# Copyright 2020 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -6,7 +6,7 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software +# Unless REQUIRED by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and diff --git a/conver_tf2npu/conver_by_ast.py b/conver_tf2npu/conver_by_ast.py index 7efaa30b8..b24488622 100644 --- a/conver_tf2npu/conver_by_ast.py +++ b/conver_tf2npu/conver_by_ast.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co.,Ltd +# Copyright 2020 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -6,7 +6,7 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software +# Unless REQUIRED by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and diff --git a/conver_tf2npu/file_op.py b/conver_tf2npu/file_op.py index 5cbc1c124..8950ad0c3 100644 --- a/conver_tf2npu/file_op.py +++ b/conver_tf2npu/file_op.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co.,Ltd +# Copyright 2020 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -6,7 +6,7 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software +# Unless REQUIRED by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and diff --git a/conver_tf2npu/main.py b/conver_tf2npu/main.py index 814f8666d..215cda8db 100644 --- a/conver_tf2npu/main.py +++ b/conver_tf2npu/main.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co.,Ltd +# Copyright 2020 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -6,7 +6,7 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software +# Unless REQUIRED by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and @@ -29,7 +29,7 @@ def para_check_and_set(argv): print('Parameter error, please check.') print(' main.py -i -o -r ') print('or: main.py --input= --output= --report=') - print('-i or --input: The source script to be converted, Default value: input/') + print('-i or --input: The source script to be converted, Default value: input/') print('-o or --output: The destination script after converted, Default value: output/') print('-r or --report: Conversion report, Default value: report/') sys.exit(2) @@ -38,7 +38,7 @@ def para_check_and_set(argv): if opt in ("-h", "--help"): print(' main.py -i -o -r ') print('or: main.py --input= --output= --report=') - print('-i or --input: The source script to be converted, Default value: input/') + print('-i or --input: The source script to be converted, Default value: input/') print('-o or --output: The destination script after converted, Default value: output/') print('-r or --report: Conversion report, Default value: report/') sys.exit() @@ -48,7 +48,7 @@ def para_check_and_set(argv): output = arg elif opt in ("-r", "--report"): report = arg - util_global.set_value('input', input) + util_global.set_value('input', input) util_global.set_value('output', output) util_global.set_value('report', report) diff --git a/conver_tf2npu/mappings/ast.json b/conver_tf2npu/mappings/ast.json index a4e793360..76b87ed13 100644 --- a/conver_tf2npu/mappings/ast.json +++ b/conver_tf2npu/mappings/ast.json @@ -1,26 +1,26 @@ { "need_conver": false, -"gelu": ["npu_unary_ops", "tf.gelu", "npu_unary_ops.gelu"], -"dropout": ["npu_ops", "tf.nn.dropout", "npu_ops.dropout"], -"init": ["print", "hvd.init", "None"], -"DistributedOptimizer": ["NPUDistributedOptimizer", "hvd.DistributedOptimizer", "NPUDistributedOptimizer"], -"rank": ["get_rank_id", "hvd.rank", "get_rank_id"], -"local_rank": ["get_local_rank_id", "hvd.local_rank", "get_local_rank_id"], -"size": ["get_rank_size", "hvd.size", "get_rank_size"], -"BroadcastGlobalVariablesHook": ["print", "BroadcastGlobalVariablesHook", "None"], -"shard": ["", "dataset.shard(xxx, xxx)", "dataset.shard(get_rank_size(), get_rank_id())"], -"EstimatorSpec": ["NPUEstimatorSpec", "tf.estimator.EstimatorSpec", "NPUEstimatorSpec"], -"RunConfig": ["NPURunConfig", "tf.estimator.RunConfig", "NPURunConfig"], -"Estimator": ["NPUEstimator", "tf.estimator.Estimator", "NPUEstimator"], +"gelu": ["npu_unary_ops", "tf.gelu", "npu_unary_ops.gelu"], +"dropout": ["npu_ops", "tf.nn.dropout", "npu_ops.dropout"], +"init": ["print", "hvd.init", "None"], +"DistributedOptimizer": ["NPUDistributedOptimizer", "hvd.DistributedOptimizer", "NPUDistributedOptimizer"], +"rank": ["get_rank_id", "hvd.rank", "get_rank_id"], +"local_rank": ["get_local_rank_id", "hvd.local_rank", "get_local_rank_id"], +"size": ["get_rank_size", "hvd.size", "get_rank_size"], +"BroadcastGlobalVariablesHook": ["print", "BroadcastGlobalVariablesHook", "None"], +"shard": ["", "dataset.shard(xxx, xxx)", "dataset.shard(get_rank_size(), get_rank_id())"], +"EstimatorSpec": ["NPUEstimatorSpec", "tf.estimator.EstimatorSpec", "NPUEstimatorSpec"], +"RunConfig": ["NPURunConfig", "tf.estimator.RunConfig", "NPURunConfig"], +"Estimator": ["NPUEstimator", "tf.estimator.Estimator", "NPUEstimator"], -"batch": ["", "batch(xxx)", "batch(xxx, drop_remainder=True)"], -"map_and_batch": ["", "map_and_batch(xxx)", "map_and_batch(xxx, drop_remainder=True)"], -"device": ["", "tf.device(xxx)", "tf.device(/cpu:0)"], +"batch": ["", "batch(xxx)", "batch(xxx, drop_remainder=True)"], +"map_and_batch": ["", "map_and_batch(xxx)", "map_and_batch(xxx, drop_remainder=True)"], +"device": ["", "tf.device(xxx)", "tf.device(/cpu:0)"], -"hvd": ["init", "rank", "local_rank", "size", "DistributedOptimizer"], -"estimator": ["Estimator", "RunConfig", "EstimatorSpec"], -"nn": ["dropout"], -"keras": ["https://support.huaweicloud.com/mprtg-A800_9000_9010/atlasprtg_13_0008.html"], +"hvd": ["init","rank", "local_rank","size", "DistributedOptimizer"], +"estimator": ["Estimator", "RunConfig", "EstimatorSpec"], +"nn": ["dropout"], +"keras": ["https://support.huaweicloud.com/mprtg-A800_9000_9010/atlasprtg_13_0008.html"], -"report_file": ["success_report.txt", "failed_report.txt", "need_migration_doc.txt"] +"report_file": ["success_report.txt", "failed_report.txt", "need_migration_doc.txt"] } \ No newline at end of file diff --git a/conver_tf2npu/util.py b/conver_tf2npu/util.py index 06879487a..5f35d919e 100644 --- a/conver_tf2npu/util.py +++ b/conver_tf2npu/util.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co.,Ltd +# Copyright 2020 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -6,7 +6,7 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software +# Unless REQUIRED by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and diff --git a/conver_tf2npu/util_global.py b/conver_tf2npu/util_global.py index e9c9c31ec..538dfcd84 100644 --- a/conver_tf2npu/util_global.py +++ b/conver_tf2npu/util_global.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co.,Ltd +# Copyright 2020 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -6,7 +6,7 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software +# Unless REQUIRED by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -- Gitee From 2c173a5037cf01ff73b93a452d66390581780dc1 Mon Sep 17 00:00:00 2001 From: yanqingshang Date: Fri, 4 Dec 2020 16:32:36 +0800 Subject: [PATCH 05/12] new tf to npu --- conver_tf2npu/ast_impl.py | 4 ++-- conver_tf2npu/conver.py | 6 +++--- conver_tf2npu/conver_by_ast.py | 4 ++-- conver_tf2npu/file_op.py | 4 ++-- conver_tf2npu/main.py | 4 ++-- conver_tf2npu/util.py | 4 ++-- conver_tf2npu/util_global.py | 4 ++-- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/conver_tf2npu/ast_impl.py b/conver_tf2npu/ast_impl.py index f341d8f01..67ece5815 100644 --- a/conver_tf2npu/ast_impl.py +++ b/conver_tf2npu/ast_impl.py @@ -4,14 +4,14 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless REQUIRED by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# ============================================================================== +# ============================================================================ import ast import util_global from util import log_success_report diff --git a/conver_tf2npu/conver.py b/conver_tf2npu/conver.py index 5ed22b424..cf4acd977 100644 --- a/conver_tf2npu/conver.py +++ b/conver_tf2npu/conver.py @@ -4,14 +4,14 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless REQUIRED by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# ============================================================================== +# ============================================================================ import os import util_global from conver_by_ast import conver_ast @@ -41,4 +41,4 @@ def conver(): else: mkdir_and_copyfile(path, abs_join(out_path, out_path_dst), file_name) - print("Finish conver, output file: " + out_path + "; report file:" + util_global.get_value('report')) + print("Finish conver, output file: " + out_path + "; report file: " + util_global.get_value('report')) diff --git a/conver_tf2npu/conver_by_ast.py b/conver_tf2npu/conver_by_ast.py index b24488622..16a1aaaf0 100644 --- a/conver_tf2npu/conver_by_ast.py +++ b/conver_tf2npu/conver_by_ast.py @@ -4,14 +4,14 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless REQUIRED by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# ============================================================================== +# ============================================================================ import os import sys import ast diff --git a/conver_tf2npu/file_op.py b/conver_tf2npu/file_op.py index 8950ad0c3..6ede41965 100644 --- a/conver_tf2npu/file_op.py +++ b/conver_tf2npu/file_op.py @@ -4,14 +4,14 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless REQUIRED by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# ============================================================================== +# ============================================================================ import os import shutil import util_global diff --git a/conver_tf2npu/main.py b/conver_tf2npu/main.py index 215cda8db..cafd62935 100644 --- a/conver_tf2npu/main.py +++ b/conver_tf2npu/main.py @@ -4,14 +4,14 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless REQUIRED by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# ============================================================================== +# ============================================================================ import sys import getopt import util_global diff --git a/conver_tf2npu/util.py b/conver_tf2npu/util.py index 5f35d919e..ae56619e9 100644 --- a/conver_tf2npu/util.py +++ b/conver_tf2npu/util.py @@ -4,14 +4,14 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless REQUIRED by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# ============================================================================== +# ============================================================================ import util_global from file_op import write_conver_report diff --git a/conver_tf2npu/util_global.py b/conver_tf2npu/util_global.py index 538dfcd84..8941003ea 100644 --- a/conver_tf2npu/util_global.py +++ b/conver_tf2npu/util_global.py @@ -4,14 +4,14 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless REQUIRED by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# ============================================================================== +# ============================================================================ import json def _init(): -- Gitee From 1215c3e7b21c7dd478c46b75befe5f96e80926c8 Mon Sep 17 00:00:00 2001 From: yanqingshang Date: Fri, 4 Dec 2020 16:46:04 +0800 Subject: [PATCH 06/12] new tf to npu --- conver_tf2npu/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conver_tf2npu/main.py b/conver_tf2npu/main.py index cafd62935..f156beb00 100644 --- a/conver_tf2npu/main.py +++ b/conver_tf2npu/main.py @@ -19,7 +19,7 @@ from file_op import before_clear from conver import conver def para_check_and_set(argv): - input = "input" + input = "input" output = "output" report = "report" -- Gitee From 3830fc3689b94680583ebb50396fca96eefd2ede Mon Sep 17 00:00:00 2001 From: yanqingshang Date: Fri, 4 Dec 2020 16:52:51 +0800 Subject: [PATCH 07/12] new tf to npu --- conver_tf2npu/ast_impl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conver_tf2npu/ast_impl.py b/conver_tf2npu/ast_impl.py index 67ece5815..e0c48a2fe 100644 --- a/conver_tf2npu/ast_impl.py +++ b/conver_tf2npu/ast_impl.py @@ -88,7 +88,7 @@ def ast_call(node): log_success_report(getattr(node, "lineno", "None"), node.func.attr) node.args = [ast.Str(s='/cpu:0')] util_global.set_value('need_conver', True) - return node + return node def insert_npu_import(r_node): npu_alias = ast.alias(name='npu_ops', asname=None) @@ -98,7 +98,7 @@ def insert_npu_import(r_node): npu_import = ast.ImportFrom(module='npu_bridge.estimator.npu_unary_ops', names=[npu_alias], level=0) r_node.body.insert(0, npu_import) -# Formate printing for locate +# Format printing for locate def node_tree(node:str): str2list = list(node.replace(' ', '')) count = 0 -- Gitee From 20dd9f234c259de7666fe4ec12ad2e8d09863b08 Mon Sep 17 00:00:00 2001 From: yanqingshang Date: Fri, 4 Dec 2020 16:56:04 +0800 Subject: [PATCH 08/12] new tf to npu --- conver_tf2npu/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conver_tf2npu/main.py b/conver_tf2npu/main.py index f156beb00..5ed7f3d04 100644 --- a/conver_tf2npu/main.py +++ b/conver_tf2npu/main.py @@ -19,7 +19,7 @@ from file_op import before_clear from conver import conver def para_check_and_set(argv): - input = "input" + input = "input" output = "output" report = "report" -- Gitee From 89f62f50bd9d319f4454bd15a9de5b26bff4e605 Mon Sep 17 00:00:00 2001 From: yanqingshang Date: Fri, 4 Dec 2020 16:58:50 +0800 Subject: [PATCH 09/12] new tf to npu --- conver_tf2npu/README.md | 59 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 conver_tf2npu/README.md diff --git a/conver_tf2npu/README.md b/conver_tf2npu/README.md new file mode 100644 index 000000000..7b215fd04 --- /dev/null +++ b/conver_tf2npu/README.md @@ -0,0 +1,59 @@ +# Tensorflow Adapter For Ascend + +[View English](README.en.md) + +Tensorflow Adapter For Ascend（简称TF_Adapter）致力于将昇腾AI处理器卓越的运算能力，便捷地提供给使用Tensorflow框架的开发者。 +开发者只需安装TF_Adapter插件，并在现有Tensorflow脚本中添加少量配置，即可实现在昇腾AI处理器上加速自己的训练任务。 + +![tfadapter](https://images.gitee.com/uploads/images/2020/1027/094640_8f305b88_8175427.jpeg "framework.jpg") + +您可以通过阅读 [TF_Adapter接口文档](https://support.huaweicloud.com/mprtg-A800_9000_9010/atlasprtg_13_0013.html) 获取更多使用细节。 +## 安装 +### 从源码安装 + +您可以从源代码构建 TF_Adapter 软件包并将其安装在昇腾AI处理器环境上。 +> TF_Adapter 插件与 Tensorflow 有严格的匹配关系，从源码构建前，您需要确保已经正确安装了 [Tensorflow v1.15.0 +>版本](https://www.tensorflow.org/install/pip) 。 + + +tfadapter也支持由源码编译，进行源码编译前，首先确保你有昇腾910 AI处理器的环境，同时系统满足以下要求： +- Linux OS +- GCC >= 7.3.0 +- CMake >= 3.14.0 +- [SWIG](http://www.swig.org/download.html) + +#### 下载源码 + +``` +git clone https://gitee.com/ascend/tensorflow.git +cd tensorflow +``` + +#### 执行脚本生成安装包 +``` +chmod +x build.sh +./build.sh +``` + +脚本执行成功后，会在output目录生成tfadapter.tar压缩文件 + +#### 安装插件包 +解压tfadapter.tar文件，生成npu_bridge-1.15.0-py3-none-any.whl, +然后使用 pip 安装 TF_Adapter 插件。 +``` +pip install npu_bridge-1.15.0-py3-none-any.whl +``` +需要注意的是， 您应当保证安装路径与您编译时指定的 python + 解释器搜索路径是一致的。 + +## 贡献 + +欢迎参与贡献。 + +## Release Notes + +Release Notes请参考[RELEASE](RELEASE.md). + +## License + +[Apache License 2.0](LICENSE) -- Gitee From 2d7daba4ed887123bdb43faf22f2b2e8de98e1ac Mon Sep 17 00:00:00 2001 From: yanqingshang Date: Fri, 4 Dec 2020 17:10:36 +0800 Subject: [PATCH 10/12] update conver_tf2npu/README.md. --- conver_tf2npu/README.md | 40 +++------------------------------------- 1 file changed, 3 insertions(+), 37 deletions(-) diff --git a/conver_tf2npu/README.md b/conver_tf2npu/README.md index 7b215fd04..afc720d8a 100644 --- a/conver_tf2npu/README.md +++ b/conver_tf2npu/README.md @@ -8,43 +8,9 @@ Tensorflow Adapter For Ascend（简称TF_Adapter）致力于将昇腾AI处理器 ![tfadapter](https://images.gitee.com/uploads/images/2020/1027/094640_8f305b88_8175427.jpeg "framework.jpg") 您可以通过阅读 [TF_Adapter接口文档](https://support.huaweicloud.com/mprtg-A800_9000_9010/atlasprtg_13_0013.html) 获取更多使用细节。 -## 安装 -### 从源码安装 - -您可以从源代码构建 TF_Adapter 软件包并将其安装在昇腾AI处理器环境上。 -> TF_Adapter 插件与 Tensorflow 有严格的匹配关系，从源码构建前，您需要确保已经正确安装了 [Tensorflow v1.15.0 ->版本](https://www.tensorflow.org/install/pip) 。 - - -tfadapter也支持由源码编译，进行源码编译前，首先确保你有昇腾910 AI处理器的环境，同时系统满足以下要求： -- Linux OS -- GCC >= 7.3.0 -- CMake >= 3.14.0 -- [SWIG](http://www.swig.org/download.html) - -#### 下载源码 - -``` -git clone https://gitee.com/ascend/tensorflow.git -cd tensorflow -``` - -#### 执行脚本生成安装包 -``` -chmod +x build.sh -./build.sh -``` - -脚本执行成功后，会在output目录生成tfadapter.tar压缩文件 - -#### 安装插件包 -解压tfadapter.tar文件，生成npu_bridge-1.15.0-py3-none-any.whl, -然后使用 pip 安装 TF_Adapter 插件。 -``` -pip install npu_bridge-1.15.0-py3-none-any.whl -``` -需要注意的是， 您应当保证安装路径与您编译时指定的 python - 解释器搜索路径是一致的。 + +脚本使用指导： +待补充 ## 贡献 -- Gitee From 70442cb5d19a2aa3310b173b763a890e79f1c6e4 Mon Sep 17 00:00:00 2001 From: yanqingshang Date: Fri, 4 Dec 2020 17:11:46 +0800 Subject: [PATCH 11/12] update tf_adapter/python/npu_bridge/npu_init.py. --- tf_adapter/python/npu_bridge/npu_init.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tf_adapter/python/npu_bridge/npu_init.py b/tf_adapter/python/npu_bridge/npu_init.py index bf5bec4c5..91dc7cf8d 100644 --- a/tf_adapter/python/npu_bridge/npu_init.py +++ b/tf_adapter/python/npu_bridge/npu_init.py @@ -4,4 +4,7 @@ from npu_bridge.estimator.npu.npu_optimizer import NPUDistributedOptimizer from npu_bridge.estimator import npu_ops from npu_bridge.estimator.npu_unary_ops import npu_unary_ops from tensorflow.core.protobuf import rewriter_config_pb2 -from tensorflow.core.protobuf.rewriter_config_pb2 import RewriterConfig \ No newline at end of file +from tensorflow.core.protobuf.rewriter_config_pb2 import RewriterConfig +from hccl.manage.api import get_local_rank_id +from hccl.manage.api import get_rank_size +from hccl.manage.api import get_rank_id \ No newline at end of file -- Gitee From 362525e31c613da7dc81092e6f5fb1db0b459d9e Mon Sep 17 00:00:00 2001 From: yanqingshang Date: Fri, 4 Dec 2020 17:12:14 +0800 Subject: [PATCH 12/12] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6=20tf?= =?UTF-8?q?=5Fadapter/python/npu=5Fbridge/hccl=5Finit.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tf_adapter/python/npu_bridge/hccl_init.py | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 tf_adapter/python/npu_bridge/hccl_init.py diff --git a/tf_adapter/python/npu_bridge/hccl_init.py b/tf_adapter/python/npu_bridge/hccl_init.py deleted file mode 100644 index b60dc5f44..000000000 --- a/tf_adapter/python/npu_bridge/hccl_init.py +++ /dev/null @@ -1,3 +0,0 @@ -from hccl.manage.api import get_local_rank_id -from hccl.manage.api import get_rank_size -from hccl.manage.api import get_rank_id \ No newline at end of file -- Gitee