From fefbb7a3cd71e6a66008ad29146abbe61a7f1a4c Mon Sep 17 00:00:00 2001
From: sunboquan <sunboquan@huawei.com>
Date: Mon, 7 Aug 2023 09:34:35 +0800
Subject: [PATCH 1/3] merge timeline

---
 precision/api_ut_tools/dump/utils.py      | 222 ++++++++++++++++++++++
 precision/api_ut_tools/run_ut/run_ut.py   |  93 +++++++++
 profiler/merge_profiling_timeline/main.py |  38 ++--
 3 files changed, 340 insertions(+), 13 deletions(-)
 create mode 100644 precision/api_ut_tools/dump/utils.py
 create mode 100644 precision/api_ut_tools/run_ut/run_ut.py

diff --git a/precision/api_ut_tools/dump/utils.py b/precision/api_ut_tools/dump/utils.py
new file mode 100644
index 0000000000..3ba951744d
--- /dev/null
+++ b/precision/api_ut_tools/dump/utils.py
@@ -0,0 +1,222 @@
+import os
+import shutil
+import sys
+from pathlib import Path
+
+from ..common.utils import print_error_log, CompareException, DumpException, Const, get_time, print_info_log, \
+    check_mode_valid, get_api_name_from_matcher
+
+from ..common.version import __version__
+
+dump_count = 0
+range_begin_flag, range_end_flag = False, False
+
+
+class DumpUtil(object):
+    dump_data_dir = None
+    dump_path = None
+    dump_switch = None
+    dump_switch_mode = Const.ALL
+    dump_switch_scope = []
+    dump_init_enable = False
+    dump_api_list = []
+    dump_filter_switch = None
+    dump_mode = Const.ALL
+    backward_input = {}
+    dump_dir_tag = 'ptdbg_dump'
+    dump_config = None
+
+    @staticmethod
+    def set_dump_path(save_path):
+        DumpUtil.dump_path = save_path
+        DumpUtil.dump_init_enable = True
+
+    @staticmethod
+    def set_dump_config(dump_config):
+        DumpUtil.dump_config = dump_config
+
+    @staticmethod
+    def set_dump_switch(switch, mode, scope, api_list, filter_switch, dump_mode):
+        DumpUtil.dump_switch = switch
+        DumpUtil.dump_switch_mode = mode
+        DumpUtil.dump_init_enable = True
+        DumpUtil.dump_switch_scope = scope
+        DumpUtil.dump_api_list = [api.lower() for api in api_list]
+        DumpUtil.dump_filter_switch = filter_switch
+        DumpUtil.dump_mode = dump_mode
+        if mode == Const.ACL:
+            DumpUtil.dump_switch_scope = [api_name.replace("backward", "forward") for api_name in scope]
+
+    def check_list_or_acl_mode(name_prefix):
+        global dump_count
+        for item in DumpUtil.dump_switch_scope:
+            if name_prefix.startswith(item):
+                dump_count = dump_count + 1
+                return True
+
+    def check_range_mode(name_prefix):
+        global range_begin_flag
+        global range_end_flag
+        if name_prefix.startswith(DumpUtil.dump_switch_scope[0]):
+            range_begin_flag = True
+            return True
+        if name_prefix.startswith(DumpUtil.dump_switch_scope[1]):
+            range_end_flag = True
+            return True
+        if range_begin_flag and not range_end_flag:
+            return True
+        return False
+
+    def check_stack_mode(name_prefix):
+        if len(DumpUtil.dump_switch_scope) == 0:
+            return True
+        elif len(DumpUtil.dump_switch_scope) == 1:
+            return name_prefix.startswith(DumpUtil.dump_switch_scope[0])
+        elif len(DumpUtil.dump_switch_scope) == 2:
+            return DumpUtil.check_range_mode(name_prefix)
+        else:
+            print_error_log("dump scope is invalid, Please set the scope mode in"
+                            " set_dump_switch with 'all', 'list', 'range', 'stack', 'acl', 'api_list'!")
+        return False
+
+    check_mapper = {
+        Const.LIST: check_list_or_acl_mode,
+        Const.ACL: check_list_or_acl_mode,
+        Const.RANGE: check_range_mode,
+        Const.STACK: check_stack_mode
+    }
+
+    @staticmethod
+    def check_switch_scope(name_prefix):
+        if DumpUtil.dump_switch_mode in DumpUtil.check_mapper:
+            check_func = DumpUtil.check_mapper[DumpUtil.dump_switch_mode]
+            return check_func(name_prefix)
+        return False
+
+    @staticmethod
+    def get_dump_path():
+        if DumpUtil.dump_path:
+            return DumpUtil.dump_path
+
+        if DumpUtil.dump_switch_mode == Const.ALL:
+            raise RuntimeError("get_dump_path: the file path is empty,"
+                               " you must use set_dump_path to set a valid dump path!!!")
+        else:
+            dir_path = os.path.realpath("./")
+            dump_file_name = "scope_dump_{}_{}_{}.pkl".format(
+                DumpUtil.dump_switch_mode, DumpUtil.dump_switch_scope[0], get_time())
+            DumpUtil.dump_path = os.path.join(dir_path, dump_file_name)
+            return DumpUtil.dump_path
+
+    @staticmethod
+    def get_dump_switch():
+        return DumpUtil.dump_switch == "ON"
+
+
+def set_dump_path(fpath=None, dump_tag='ptdbg_dump'):
+    if fpath is None:
+        raise RuntimeError("set_dump_path '{}' error, please set a valid filename".format(fpath))
+        return
+    real_path = os.path.realpath(fpath)
+    if not os.path.isdir(real_path):
+        print_error_log(
+            "set_dump_path '{}' error, the path is not a directory please set a valid directory.".format(real_path))
+        raise DumpException(DumpException.INVALID_PATH_ERROR)
+    DumpUtil.set_dump_path(real_path)
+    DumpUtil.dump_dir_tag = dump_tag
+
+
+def generate_dump_path_str():
+    if DumpUtil.dump_switch_mode == 'acl':
+        if DumpUtil.dump_config == '':
+            print_error_log("Please provide dump config for register hook before turning on dump switch!")
+            raise DumpException(DumpException.NONE_ERROR)
+        dump_path = f"according to dump config {DumpUtil.dump_config}"
+    else:
+        dump_path = f"to {DumpUtil.dump_path}"
+    return dump_path
+
+
+def set_dump_switch(switch, mode=Const.ALL, scope=[], api_list=[], filter_switch=Const.ON, dump_mode=Const.ALL):
+    try:
+        check_mode_valid(mode)
+        assert switch in ["ON", "OFF"], "Please set dump switch with 'ON' or 'OFF'."
+        assert filter_switch in ["ON", "OFF"], "Please set filter_switch with 'ON' or 'OFF'."
+        assert dump_mode in ["all", "forward", "backward"], \
+            "Please set dump_mode with 'all' or 'forward' or 'backward'."
+        if mode == Const.RANGE:
+            assert len(scope) == 2, "set_dump_switch, scope param set invalid, it's must be [start, end]."
+        if mode == Const.LIST:
+            assert len(scope) != 0, "set_dump_switch, scope param set invalid, it's should not be an empty list."
+        if mode == Const.STACK:
+            assert len(scope) <= 2, "set_dump_switch, scope param set invalid, it's must be [start, end] or []."
+        if mode == Const.ACL:
+            assert len(scope) == 1, \
+                "set_dump_switch, scope param set invalid, only one api name is supported in acl mode."
+        if mode == Const.API_LIST:
+            assert isinstance(api_list, list) and len(api_list) >= 1, \
+                "Current dump mode is 'api_list', but the content of api_list parameter is empty or valid."
+    except (CompareException, AssertionError) as err:
+        print_error_log(str(err))
+        sys.exit()
+
+    if switch == "OFF":
+        dump_path_str = generate_dump_path_str()
+    DumpUtil.set_dump_switch(switch, mode=mode, scope=scope, api_list=api_list,
+                             filter_switch=filter_switch, dump_mode=dump_mode)
+    if switch == "ON":
+        dump_path_str = generate_dump_path_str()
+
+    global dump_count
+    if switch == "ON":
+        print_info_log(f"Dump switch is turned on. Dump data will be saved {dump_path_str}. ")
+        if mode == Const.LIST:
+            dump_count = 0
+    else:
+        print_info_log(f"Dump switch is turned off. Dump data has been saved {dump_path_str}. ")
+        if mode == Const.LIST:
+            print_info_log("The number of matched dump is {}".format(dump_count))
+
+def _set_dump_switch4api_list(name):
+    if DumpUtil.dump_api_list:
+        api_name = get_api_name_from_matcher(name)
+        DumpUtil.dump_switch = "ON" if api_name in DumpUtil.dump_api_list else "OFF"
+
+
+def set_backward_input(backward_input):
+    for index, api_name in enumerate(DumpUtil.dump_switch_scope):
+        DumpUtil.backward_input[api_name] = backward_input[index]
+
+
+def make_dump_data_dir(dump_file_name):
+    dump_path, file_name = os.path.split(os.path.realpath(dump_file_name))
+    name_body, name_extension = os.path.splitext(file_name)
+    output_dir = os.path.join(dump_path, f"{name_body}")
+    if not os.path.exists(output_dir):
+        os.mkdir(output_dir, mode=0o750)
+    else:
+        shutil.rmtree(output_dir, ignore_errors=True)
+        os.mkdir(output_dir, mode=0o750)
+    return output_dir
+
+
+def make_dump_dirs(rank):
+    dump_file_name, dump_file_name_body = "dump.pkl", "dump"
+    dump_root_dir = DumpUtil.dump_path if DumpUtil.dump_path else "./"
+    tag_dir = os.path.join(dump_root_dir, DumpUtil.dump_dir_tag + f'_v{__version__}')
+    Path(tag_dir).mkdir(mode=0o750, parents=True, exist_ok=True)
+    rank_dir = os.path.join(tag_dir, 'rank' + str(rank))
+    if not os.path.exists(rank_dir):
+        os.mkdir(rank_dir, mode=0o750)
+    DumpUtil.dump_dir = rank_dir
+    dump_file_path = os.path.join(rank_dir, dump_file_name)
+    DumpUtil.set_dump_path(dump_file_path)
+
+
+def check_writable(dump_file):
+    if not os.access(dump_file, os.W_OK):
+        print_error_log(
+            'The path {} does not have permission to write. Please check the path permission'.format(
+                dump_file))
+        raise DumpException(DumpException.INVALID_PATH_ERROR)
+
diff --git a/precision/api_ut_tools/run_ut/run_ut.py b/precision/api_ut_tools/run_ut/run_ut.py
new file mode 100644
index 0000000000..5cb17d8c40
--- /dev/null
+++ b/precision/api_ut_tools/run_ut/run_ut.py
@@ -0,0 +1,93 @@
+# 用户构造并运行api用例，注意前反向的区分
+import yaml
+import os
+import json
+import torch
+
+FLOAT_TYPE = ['torch.float32', 'torch.float', 'torch.float64', 'torch.double', 'torch.float16', \
+            'torch.half', 'torch.bfloat16']
+
+cur_path = os.path.dirname(os.path.realpath(__file__))
+yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml")
+with open(yaml_path, 'r') as f:
+    WrapFunctionalOps = yaml.safe_load(f).get('functional')
+
+for f in dir(torch.nn.functional):
+    locals().update({f: getattr(torch.nn.functional, f)})
+
+
+def run_ut():
+    print("start")
+    forward_pkl = open("/home/wangchao/torch_test/dump_data_new/npu/ptdbg_dump_v1.0/rank0/dump.pkl")
+    backward_pkl = open("/home/wangchao/torch_test/dump_data_new/npu/ptdbg_dump_v1.0/rank0/dump_backward.pkl")
+    forward_content = forward_pkl.readlines()
+    backward_content = backward_pkl.readlines()
+    for api_info in forward_content:
+        api_json = json.loads(api_info)
+        for key, value in api_json.items():
+            [api_type, api_name, index, mode] = key.split("*")
+            print(api_name)
+            api_feature = key.rsplit("*", 1)[0]
+            args, kwargs = generate_input(value.get("args"), api_json.get("kwargs"))
+            if api_type == "Functional":
+                out = eval(api_name)(*args, **kwargs)
+            if api_type == "Tensor":
+                out = getattr(torch._C._TensorBase, str(api_name))(*args, **kwargs)
+            for line in backward_content:
+                if api_feature in line:
+                    api_back_json = json.loads(line)
+                    for params in api_back_json.values():
+                        grad = nested_generate_input(params.get("args"), True, False)
+                        out.backward(grad)
+            input_grad = [tensor.grad for tensor in args if isinstance(tensor, torch.Tensor)]
+            print("forward")
+            print(out)
+            print("backward")
+            print(input_grad)
+
+
+def generate_input(input_args, input_kwargs, need_backward=True, need_convert=False):  # 没有考虑dict of tensor
+    args = []
+    kwargs = {}
+
+    for info in input_args:
+        args.append(nested_generate_input(info, need_backward, need_convert))
+    if kwargs:
+        for key, info in input_kwargs.items():
+            kwargs[key] = nested_generate_input(info, need_backward, need_convert)
+    return args, kwargs
+
+
+def nested_generate_input(info, need_backward, need_convert):
+    if isinstance(info, list):
+        result = []
+        for i in info:
+            result.append(nested_generate_input(i, need_backward, need_convert))
+        return result
+        # return list(map(nested_generate_input, info))
+    # elif isinstance(input_info, tuple):
+    #     return tuple(map(generate_input, input_info))
+    else:
+        if info['type'] == 'torch.Tensor':
+            low, high = info['Min'], info['Max']
+            data_dtype = info['dtype']
+            if data_dtype in FLOAT_TYPE: #应该搞个float类型列表
+                if need_convert and data_dtype == "torch.float16":
+                    data_dtype = "torch.float32"
+                scale = high - low
+                rand01 = torch.rand(tuple(info['shape']), dtype=eval(data_dtype))
+                inpt = rand01 * scale + low
+                if need_backward:
+                    inpt.requires_grad_(True)
+                    inpt.retain_grad()
+            elif 'int' in data_dtype or 'long' in data_dtype: # 应该搞个int类型列表,
+                inpt = torch.randint(int(low), int(high)+1, tuple(info['shape']),
+                                     dtype=eval(data_dtype)) # high + 1因为右边是开区间
+            else:
+                print(f'Warning: Dtype is not supported: ', info['dtype'])
+                raise NotImplementedError()
+        else:
+            inpt = info['value'] # 遗留问题：需要考虑是否要转换成原本类型
+        return inpt
+
+run_ut()
\ No newline at end of file
diff --git a/profiler/merge_profiling_timeline/main.py b/profiler/merge_profiling_timeline/main.py
index 84a0f7d71b..16afee19b2 100644
--- a/profiler/merge_profiling_timeline/main.py
+++ b/profiler/merge_profiling_timeline/main.py
@@ -70,6 +70,7 @@ def get_timeline_info(args, prof_dirs):
 
         # 从info.json读取rank_id
         rank_id = get_rank_id_from_info_json(pro_path)
+        ts_difference_us = get_absolute_ts_start_info(pro_path)
         if rank_id is None:
             print(f"WARN, There is not rank id info in {pro_path}")
             continue
@@ -77,7 +78,7 @@ def get_timeline_info(args, prof_dirs):
         timeline_path = get_timeline_path(pro_path, args.type)
 
         if os.path.exists(timeline_path):
-            timeline_info[rank_id] = timeline_path
+            timeline_info[rank_id] = (timeline_path, ts_difference_us)
         else:
             print(f"WARN, The file \"{timeline_path}\" does not exist.")
     return timeline_info
@@ -96,6 +97,20 @@ def get_timeline_path(pro_path, type):
                 return timeline_path
     return
 
+def get_absolute_ts_start_info(pro_path) -> float:
+    for root, dirs, files in os.walk(pro_path):
+        for file in files:
+            if "start_json" in file and ".done" not in file:
+                start_json = os.path.join(root, file)
+                break
+        if start_json:
+            with open(start_json, "r+") as f:
+                info = json.load(f)
+            ts_us = float(info.get("collectionTimeBegin"), 0)
+            ts_ns = float(info.get("clockMonotonicRaw"), 0)
+            if ts_us and ts_ns:
+                return 0
+        return ts_us-ts_ns/1000
 
 def get_rank_id_from_info_json(pro_path):
     info_json = ""
@@ -119,8 +134,6 @@ def merge_timeline_general(args):
     timeline_info = get_timeline_info(args, prof_dir)
     timeline_files_dict = {}
 
-    node_time_diff = get_node_time_diff(args.timediff) if args.timediff else None
-
     # 合并部分profiling items
     process_list = args.items.split(",") if args.items else None
 
@@ -132,7 +145,7 @@ def merge_timeline_general(args):
 
     for rank_id in rank_ids:
         timeline_files_dict[rank_id] = timeline_info.get(rank_id)
-    merge_timeline_events(timeline_files_dict, process_list, node_time_diff)
+    merge_timeline_events(timeline_files_dict, process_list)
 
 
 def merge_timeline_custom(args):
@@ -140,25 +153,24 @@ def merge_timeline_custom(args):
     timeline_files = natural_sort(os.listdir(args.data))
     timeline_files_dict = {}
     for idx, timeline_file in enumerate(timeline_files):
-        timeline_files_dict[idx] = os.path.join(args.data, timeline_file)
-    node_time_diff = get_node_time_diff(args.timediff) if args.timediff else None
+        timeline_files_dict[idx] = (os.path.join(args.data, timeline_file),0)
     # 合并部分profiling items
     process_list = args.items.split(",") if args.items else None
-    merge_timeline_events(timeline_files_dict, process_list, node_time_diff)
+    merge_timeline_events(timeline_files_dict, process_list)
 
 
-def merge_timeline_events(timeline_file_dict, process_list, node_time_diff=None):
+def merge_timeline_events(timeline_file_dict, process_list):
     """
     输入需要合并的timeline文件路径及对应的rank_id/id、需要合并的process_list、校准时间差node_time_diff
     输出合并timeline
     """
     new_events = []
-    for rank_id, timeline_file_path in timeline_file_dict.items():
+    for rank_id, data_tuple in timeline_file_dict.items():
+        timeline_file_path = data_tuple[0]
+        ts_difference_us = data_tuple[1]
         node = rank_id // 8
         print("rank id: ", rank_id, "timeline file: ", timeline_file_path)
 
-        # 获取相应的时间差异
-        node_time = node_time_diff[node] if node_time_diff else None
         try:
             with open(timeline_file_path, 'r+') as f:
                 cur_events = json.load(f)
@@ -189,8 +201,8 @@ def merge_timeline_events(timeline_file_dict, process_list, node_time_diff=None)
                 continue
 
             # 当前节点间时间误差可用时，进行时间校准
-            if event.get("ts") and node_time:
-                event["ts"] = event["ts"] - node_time * 1000000
+            if event.get("ts") and ts_difference_us:
+                event["ts"] = event["ts"] + ts_difference_us
 
             # 区分不同rank的同一进程的pid
             if isinstance(event.get("pid"), (str, int)):
-- 
Gitee


From 3b53bbf6c80a8bf7d667bdc1b528c6a5bada4d06 Mon Sep 17 00:00:00 2001
From: sunboquan <sunboquan@huawei.com>
Date: Mon, 7 Aug 2023 11:41:11 +0800
Subject: [PATCH 2/3] merge timeline

---
 profiler/merge_profiling_timeline/main.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/profiler/merge_profiling_timeline/main.py b/profiler/merge_profiling_timeline/main.py
index 16afee19b2..eac48f5683 100644
--- a/profiler/merge_profiling_timeline/main.py
+++ b/profiler/merge_profiling_timeline/main.py
@@ -100,17 +100,17 @@ def get_timeline_path(pro_path, type):
 def get_absolute_ts_start_info(pro_path) -> float:
     for root, dirs, files in os.walk(pro_path):
         for file in files:
-            if "start_json" in file and ".done" not in file:
+            if "start_info" in file and ".done" not in file:
                 start_json = os.path.join(root, file)
                 break
-        if start_json:
-            with open(start_json, "r+") as f:
-                info = json.load(f)
-            ts_us = float(info.get("collectionTimeBegin"), 0)
-            ts_ns = float(info.get("clockMonotonicRaw"), 0)
-            if ts_us and ts_ns:
-                return 0
-        return ts_us-ts_ns/1000
+    if start_json:
+        with open(start_json, "r+") as f:
+            info = json.load(f)
+        ts_us = float(info.get("collectionTimeBegin"), 0)
+        ts_ns = float(info.get("clockMonotonicRaw"), 0)
+        if not ts_us and not ts_ns:
+            return 0
+    return ts_us-ts_ns/1000
 
 def get_rank_id_from_info_json(pro_path):
     info_json = ""
-- 
Gitee


From fe2c11aa2afe3698d4cd37cf158a46aa4280a88b Mon Sep 17 00:00:00 2001
From: sunboquan <sunboquan@huawei.com>
Date: Tue, 8 Aug 2023 20:47:17 +0800
Subject: [PATCH 3/3] License append

---
 Third_Party_Open_Source_Software_Notice       | 39 +++++++++++++++++++
 profiler/cluster_analyse/__init__.py          | 14 +++++++
 .../analysis/analysis_facade.py               | 15 +++++++
 profiler/cluster_analyse/cluster_analysis.py  | 15 +++++++
 .../cluster_data_preprocess/__init__.py       | 14 +++++++
 .../data_preprocessor.py                      | 15 +++++++
 .../pytorch_data_preprocessor.py              | 15 +++++++
 .../cluster_op_summary_analysis.py            |  7 ++--
 .../communication_group_generator.py          | 15 +++++++
 profiler/compare_tools/__init__.py            | 14 +++++++
 profiler/compare_tools/torch_op_compare.py    | 15 +++++++
 .../distribute_modify_hostname.bash           | 15 +++++++
 profiler/performance_analyse/__init__.py      | 14 +++++++
 profiler/performance_analyse/gpu_parser.py    | 15 +++++++
 profiler/performance_analyse/npu_parser.py    | 15 +++++++
 profiler/performance_analyse/parser_helper.py | 15 +++++++
 .../performance_analyse/profiling_parse.py    | 15 +++++++
 17 files changed, 263 insertions(+), 4 deletions(-)
 create mode 100644 Third_Party_Open_Source_Software_Notice

diff --git a/Third_Party_Open_Source_Software_Notice b/Third_Party_Open_Source_Software_Notice
new file mode 100644
index 0000000000..7d9870cf15
--- /dev/null
+++ b/Third_Party_Open_Source_Software_Notice
@@ -0,0 +1,39 @@
+OPEN SOURCE SOFTWARE NOTICE
+
+Please note we provide an open source software notice along with this product and/or this product firmware (in the following just “this product”). The open source software licenses are granted by the respective right holders. And the open source licenses prevail all other license information with regard to the respective open source software contained in the product, including but not limited to End User Software Licensing Agreement. This notice is provided on behalf of Huawei Technologies Co. Ltd. and any of its local subsidiaries which may have provided this product to you in your local country.
+
+Warranty Disclaimer
+THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS.
+
+Copyright Notice and License Texts
+Software: For Kineto software
+
+Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.
+
+All contributions by Microsoft:
+Copyright (c) Microsoft Corporation. (The Azure AI Platform team)
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+ * Neither the name Facebook nor the names of its contributors may be used to
+   endorse or promote products derived from this software without specific
+   prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/profiler/cluster_analyse/__init__.py b/profiler/cluster_analyse/__init__.py
index e69de29bb2..1af7850112 100644
--- a/profiler/cluster_analyse/__init__.py
+++ b/profiler/cluster_analyse/__init__.py
@@ -0,0 +1,14 @@
+# Copyright (c) 2023, Huawei Technologies.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/profiler/cluster_analyse/analysis/analysis_facade.py b/profiler/cluster_analyse/analysis/analysis_facade.py
index 9368352263..98526b94ab 100644
--- a/profiler/cluster_analyse/analysis/analysis_facade.py
+++ b/profiler/cluster_analyse/analysis/analysis_facade.py
@@ -1,3 +1,18 @@
+# Copyright (c) 2023, Huawei Technologies.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 class AnalysisFacade:
     analysis_module = {}
 
diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py
index f9cb1b276d..d6ac6a5716 100644
--- a/profiler/cluster_analyse/cluster_analysis.py
+++ b/profiler/cluster_analyse/cluster_analysis.py
@@ -1,3 +1,18 @@
+# Copyright (c) 2023, Huawei Technologies.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import argparse
 from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor
 from communication_group.communication_group_generator import CommunicationGroupGenerator
diff --git a/profiler/cluster_analyse/cluster_data_preprocess/__init__.py b/profiler/cluster_analyse/cluster_data_preprocess/__init__.py
index e69de29bb2..1af7850112 100644
--- a/profiler/cluster_analyse/cluster_data_preprocess/__init__.py
+++ b/profiler/cluster_analyse/cluster_data_preprocess/__init__.py
@@ -0,0 +1,14 @@
+# Copyright (c) 2023, Huawei Technologies.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/profiler/cluster_analyse/cluster_data_preprocess/data_preprocessor.py b/profiler/cluster_analyse/cluster_data_preprocess/data_preprocessor.py
index 947826ce1c..c2e4bd4789 100644
--- a/profiler/cluster_analyse/cluster_data_preprocess/data_preprocessor.py
+++ b/profiler/cluster_analyse/cluster_data_preprocess/data_preprocessor.py
@@ -1,3 +1,18 @@
+# Copyright (c) 2023, Huawei Technologies.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from abc import abstractmethod
 
 
diff --git a/profiler/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py b/profiler/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py
index 4a630c89ee..2a3492697b 100644
--- a/profiler/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py
+++ b/profiler/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py
@@ -1,3 +1,18 @@
+# Copyright (c) 2023, Huawei Technologies.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
 
 
diff --git a/profiler/cluster_analyse/cluster_kernels_analysis/cluster_op_summary_analysis.py b/profiler/cluster_analyse/cluster_kernels_analysis/cluster_op_summary_analysis.py
index cdd4b9e0ef..db9494734e 100644
--- a/profiler/cluster_analyse/cluster_kernels_analysis/cluster_op_summary_analysis.py
+++ b/profiler/cluster_analyse/cluster_kernels_analysis/cluster_op_summary_analysis.py
@@ -1,7 +1,7 @@
-#!/bin/bash
-# Copyright 2023 Huawei Technologies Co., Ltd
+# Copyright (c) 2023, Huawei Technologies.
+# All rights reserved.
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Licensed under the Apache License, Version 2.0  (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 from pathlib import Path
 
 import pandas as pd
diff --git a/profiler/cluster_analyse/communication_group/communication_group_generator.py b/profiler/cluster_analyse/communication_group/communication_group_generator.py
index 273de70a09..b287f07eda 100644
--- a/profiler/cluster_analyse/communication_group/communication_group_generator.py
+++ b/profiler/cluster_analyse/communication_group/communication_group_generator.py
@@ -1,3 +1,18 @@
+# Copyright (c) 2023, Huawei Technologies.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 class CommunicationGroupGenerator:
     def __init__(self, collection_path: str, data_map: dict):
         self.collection_path = collection_path
diff --git a/profiler/compare_tools/__init__.py b/profiler/compare_tools/__init__.py
index e69de29bb2..1af7850112 100644
--- a/profiler/compare_tools/__init__.py
+++ b/profiler/compare_tools/__init__.py
@@ -0,0 +1,14 @@
+# Copyright (c) 2023, Huawei Technologies.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/profiler/compare_tools/torch_op_compare.py b/profiler/compare_tools/torch_op_compare.py
index 7d32a03e3f..1c4551aa4b 100644
--- a/profiler/compare_tools/torch_op_compare.py
+++ b/profiler/compare_tools/torch_op_compare.py
@@ -1,3 +1,18 @@
+# Copyright (c) 2023, Huawei Technologies.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import argparse
 import ast
 import copy
diff --git a/profiler/distribute_tools/distribute_modify_hostname.bash b/profiler/distribute_tools/distribute_modify_hostname.bash
index c835ea51dc..d47b035499 100644
--- a/profiler/distribute_tools/distribute_modify_hostname.bash
+++ b/profiler/distribute_tools/distribute_modify_hostname.bash
@@ -1,3 +1,18 @@
+# Copyright (c) 2023, Huawei Technologies.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 #!/bin/bash
 
 SSH="ssh -o StrictHostKeyChecking=no"
diff --git a/profiler/performance_analyse/__init__.py b/profiler/performance_analyse/__init__.py
index e69de29bb2..1af7850112 100644
--- a/profiler/performance_analyse/__init__.py
+++ b/profiler/performance_analyse/__init__.py
@@ -0,0 +1,14 @@
+# Copyright (c) 2023, Huawei Technologies.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/profiler/performance_analyse/gpu_parser.py b/profiler/performance_analyse/gpu_parser.py
index 977d7a1d79..e95eb13801 100644
--- a/profiler/performance_analyse/gpu_parser.py
+++ b/profiler/performance_analyse/gpu_parser.py
@@ -1,3 +1,18 @@
+# Copyright (c) 2023, Huawei Technologies.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from collections import Counter, defaultdict
 import pandas as pd
 
diff --git a/profiler/performance_analyse/npu_parser.py b/profiler/performance_analyse/npu_parser.py
index a3c74917cc..e2c8dbc0d2 100644
--- a/profiler/performance_analyse/npu_parser.py
+++ b/profiler/performance_analyse/npu_parser.py
@@ -1,3 +1,18 @@
+# Copyright (c) 2023, Huawei Technologies.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import sys
 import pandas as pd
 from collections import defaultdict
diff --git a/profiler/performance_analyse/parser_helper.py b/profiler/performance_analyse/parser_helper.py
index 6b91fdd534..9cfab6910d 100644
--- a/profiler/performance_analyse/parser_helper.py
+++ b/profiler/performance_analyse/parser_helper.py
@@ -1,3 +1,18 @@
+# Copyright (c) 2023, Huawei Technologies.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import json
 import os
 
diff --git a/profiler/performance_analyse/profiling_parse.py b/profiler/performance_analyse/profiling_parse.py
index 2f77349534..0e81f2d219 100644
--- a/profiler/performance_analyse/profiling_parse.py
+++ b/profiler/performance_analyse/profiling_parse.py
@@ -1,3 +1,18 @@
+# Copyright (c) 2023, Huawei Technologies.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import argparse
 import os
 
-- 
Gitee