diff --git a/Third_Party_Open_Source_Software_Notice b/Third_Party_Open_Source_Software_Notice new file mode 100644 index 0000000000000000000000000000000000000000..7d9870cf1557aa0918d0a599acd333327989b05b --- /dev/null +++ b/Third_Party_Open_Source_Software_Notice @@ -0,0 +1,39 @@ +OPEN SOURCE SOFTWARE NOTICE + +Please note we provide an open source software notice along with this product and/or this product firmware (in the following just “this product”). The open source software licenses are granted by the respective right holders. And the open source licenses prevail all other license information with regard to the respective open source software contained in the product, including but not limited to End User Software Licensing Agreement. This notice is provided on behalf of Huawei Technologies Co. Ltd. and any of its local subsidiaries which may have provided this product to you in your local country. + +Warranty Disclaimer +THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + +Copyright Notice and License Texts +Software: For Kineto software + +Copyright (c) Facebook, Inc. and its affiliates. All rights reserved. + +All contributions by Microsoft: +Copyright (c) Microsoft Corporation. (The Azure AI Platform team) + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/profiler/cluster_analyse/__init__.py b/profiler/cluster_analyse/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..1af7850112ed8824e5f9b73b9dbab54215f9c9cd 100644 --- a/profiler/cluster_analyse/__init__.py +++ b/profiler/cluster_analyse/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2023, Huawei Technologies. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/profiler/cluster_analyse/analysis/analysis_facade.py b/profiler/cluster_analyse/analysis/analysis_facade.py index 93683522636087057f3874082d55187c13ffd36a..98526b94ab16d794f3a765ded9cf6187c9881493 100644 --- a/profiler/cluster_analyse/analysis/analysis_facade.py +++ b/profiler/cluster_analyse/analysis/analysis_facade.py @@ -1,3 +1,18 @@ +# Copyright (c) 2023, Huawei Technologies. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + class AnalysisFacade: analysis_module = {} diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index f9cb1b276d9320fca6e49654a42435e4a78ae4a1..d6ac6a57169c9059974939cdbc19c8a6e951891e 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -1,3 +1,18 @@ +# Copyright (c) 2023, Huawei Technologies. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import argparse from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor from communication_group.communication_group_generator import CommunicationGroupGenerator diff --git a/profiler/cluster_analyse/cluster_data_preprocess/__init__.py b/profiler/cluster_analyse/cluster_data_preprocess/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..1af7850112ed8824e5f9b73b9dbab54215f9c9cd 100644 --- a/profiler/cluster_analyse/cluster_data_preprocess/__init__.py +++ b/profiler/cluster_analyse/cluster_data_preprocess/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2023, Huawei Technologies. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/profiler/cluster_analyse/cluster_data_preprocess/data_preprocessor.py b/profiler/cluster_analyse/cluster_data_preprocess/data_preprocessor.py index 947826ce1cb6815e2af0c15e33f2e927241ef85b..c2e4bd4789fc10a11a19e2e4667f48c231824540 100644 --- a/profiler/cluster_analyse/cluster_data_preprocess/data_preprocessor.py +++ b/profiler/cluster_analyse/cluster_data_preprocess/data_preprocessor.py @@ -1,3 +1,18 @@ +# Copyright (c) 2023, Huawei Technologies. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from abc import abstractmethod diff --git a/profiler/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py b/profiler/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py index 4a630c89ee1171083f1c786db3c99f4232df5229..2a3492697b52eb72159f82c95d799f05b83d786c 100644 --- a/profiler/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py +++ b/profiler/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py @@ -1,3 +1,18 @@ +# Copyright (c) 2023, Huawei Technologies. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os diff --git a/profiler/cluster_analyse/cluster_kernels_analysis/cluster_op_summary_analysis.py b/profiler/cluster_analyse/cluster_kernels_analysis/cluster_op_summary_analysis.py index e35c6a8a972f99dcd45153bf11161667d20c9ca4..a95ac0a01807d9a163f8992d7df8b2f294fca8fb 100644 --- a/profiler/cluster_analyse/cluster_kernels_analysis/cluster_op_summary_analysis.py +++ b/profiler/cluster_analyse/cluster_kernels_analysis/cluster_op_summary_analysis.py @@ -1,3 +1,18 @@ +# Copyright (c) 2023, Huawei Technologies. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from pathlib import Path import pandas as pd diff --git a/profiler/cluster_analyse/communication_group/communication_group_generator.py b/profiler/cluster_analyse/communication_group/communication_group_generator.py index 273de70a09bc576d17a1f8b94021bb16a7b2a8e6..b287f07eda2840c905fde8357597f4f60d638391 100644 --- a/profiler/cluster_analyse/communication_group/communication_group_generator.py +++ b/profiler/cluster_analyse/communication_group/communication_group_generator.py @@ -1,3 +1,18 @@ +# Copyright (c) 2023, Huawei Technologies. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + class CommunicationGroupGenerator: def __init__(self, collection_path: str, data_map: dict): self.collection_path = collection_path diff --git a/profiler/compare_tools/__init__.py b/profiler/compare_tools/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..1af7850112ed8824e5f9b73b9dbab54215f9c9cd 100644 --- a/profiler/compare_tools/__init__.py +++ b/profiler/compare_tools/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2023, Huawei Technologies. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/profiler/compare_tools/torch_op_compare.py b/profiler/compare_tools/torch_op_compare.py index 7d32a03e3f2d0fca02cae8c2955af8a68098c3b8..1c4551aa4b221054d3fe921a66b46c7a65c1048a 100644 --- a/profiler/compare_tools/torch_op_compare.py +++ b/profiler/compare_tools/torch_op_compare.py @@ -1,3 +1,18 @@ +# Copyright (c) 2023, Huawei Technologies. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import argparse import ast import copy diff --git a/profiler/distribute_tools/distribute_modify_hostname.bash b/profiler/distribute_tools/distribute_modify_hostname.bash index c835ea51dcaf3132a203312aef3346d094229e96..d47b035499e678ae969f0e5a0be12802fa571155 100644 --- a/profiler/distribute_tools/distribute_modify_hostname.bash +++ b/profiler/distribute_tools/distribute_modify_hostname.bash @@ -1,3 +1,18 @@ +# Copyright (c) 2023, Huawei Technologies. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + #!/bin/bash SSH="ssh -o StrictHostKeyChecking=no" diff --git a/profiler/merge_profiling_timeline/main.py b/profiler/merge_profiling_timeline/main.py index 84a0f7d71bf7b15f7d0fdac37b3a3d7fa73d2515..eac48f5683961ae4a139dea5daaee049e406784c 100644 --- a/profiler/merge_profiling_timeline/main.py +++ b/profiler/merge_profiling_timeline/main.py @@ -70,6 +70,7 @@ def get_timeline_info(args, prof_dirs): # 从info.json读取rank_id rank_id = get_rank_id_from_info_json(pro_path) + ts_difference_us = get_absolute_ts_start_info(pro_path) if rank_id is None: print(f"WARN, There is not rank id info in {pro_path}") continue @@ -77,7 +78,7 @@ def get_timeline_info(args, prof_dirs): timeline_path = get_timeline_path(pro_path, args.type) if os.path.exists(timeline_path): - timeline_info[rank_id] = timeline_path + timeline_info[rank_id] = (timeline_path, ts_difference_us) else: print(f"WARN, The file \"{timeline_path}\" does not exist.") return timeline_info @@ -96,6 +97,20 @@ def get_timeline_path(pro_path, type): return timeline_path return +def get_absolute_ts_start_info(pro_path) -> float: + for root, dirs, files in os.walk(pro_path): + for file in files: + if "start_info" in file and ".done" not in file: + start_json = os.path.join(root, file) + break + if start_json: + with open(start_json, "r+") as f: + info = json.load(f) + ts_us = float(info.get("collectionTimeBegin"), 0) + ts_ns = float(info.get("clockMonotonicRaw"), 0) + if not ts_us and not ts_ns: + return 0 + return ts_us-ts_ns/1000 def get_rank_id_from_info_json(pro_path): info_json = "" @@ -119,8 +134,6 @@ def merge_timeline_general(args): timeline_info = get_timeline_info(args, prof_dir) timeline_files_dict = {} - node_time_diff = get_node_time_diff(args.timediff) if args.timediff else None - # 合并部分profiling items process_list = args.items.split(",") if args.items else None @@ -132,7 +145,7 @@ def merge_timeline_general(args): for rank_id in rank_ids: timeline_files_dict[rank_id] = timeline_info.get(rank_id) - merge_timeline_events(timeline_files_dict, process_list, node_time_diff) + merge_timeline_events(timeline_files_dict, process_list) def merge_timeline_custom(args): @@ -140,25 +153,24 @@ def merge_timeline_custom(args): timeline_files = natural_sort(os.listdir(args.data)) timeline_files_dict = {} for idx, timeline_file in enumerate(timeline_files): - timeline_files_dict[idx] = os.path.join(args.data, timeline_file) - node_time_diff = get_node_time_diff(args.timediff) if args.timediff else None + timeline_files_dict[idx] = (os.path.join(args.data, timeline_file),0) # 合并部分profiling items process_list = args.items.split(",") if args.items else None - merge_timeline_events(timeline_files_dict, process_list, node_time_diff) + merge_timeline_events(timeline_files_dict, process_list) -def merge_timeline_events(timeline_file_dict, process_list, node_time_diff=None): +def merge_timeline_events(timeline_file_dict, process_list): """ 输入需要合并的timeline文件路径及对应的rank_id/id、需要合并的process_list、校准时间差node_time_diff 输出合并timeline """ new_events = [] - for rank_id, timeline_file_path in timeline_file_dict.items(): + for rank_id, data_tuple in timeline_file_dict.items(): + timeline_file_path = data_tuple[0] + ts_difference_us = data_tuple[1] node = rank_id // 8 print("rank id: ", rank_id, "timeline file: ", timeline_file_path) - # 获取相应的时间差异 - node_time = node_time_diff[node] if node_time_diff else None try: with open(timeline_file_path, 'r+') as f: cur_events = json.load(f) @@ -189,8 +201,8 @@ def merge_timeline_events(timeline_file_dict, process_list, node_time_diff=None) continue # 当前节点间时间误差可用时,进行时间校准 - if event.get("ts") and node_time: - event["ts"] = event["ts"] - node_time * 1000000 + if event.get("ts") and ts_difference_us: + event["ts"] = event["ts"] + ts_difference_us # 区分不同rank的同一进程的pid if isinstance(event.get("pid"), (str, int)): diff --git a/profiler/performance_analyse/__init__.py b/profiler/performance_analyse/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..1af7850112ed8824e5f9b73b9dbab54215f9c9cd 100644 --- a/profiler/performance_analyse/__init__.py +++ b/profiler/performance_analyse/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2023, Huawei Technologies. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/profiler/performance_analyse/gpu_parser.py b/profiler/performance_analyse/gpu_parser.py index 977d7a1d79ea0d4db948b039731041b98ba3e18f..e95eb1380128cd4dc0a26b4b1ab5077425667a22 100644 --- a/profiler/performance_analyse/gpu_parser.py +++ b/profiler/performance_analyse/gpu_parser.py @@ -1,3 +1,18 @@ +# Copyright (c) 2023, Huawei Technologies. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from collections import Counter, defaultdict import pandas as pd diff --git a/profiler/performance_analyse/npu_parser.py b/profiler/performance_analyse/npu_parser.py index a3c74917cc42d3ea49a62a2a4086f10052e2010f..e2c8dbc0d286c9e28cc3927d821e449792a91670 100644 --- a/profiler/performance_analyse/npu_parser.py +++ b/profiler/performance_analyse/npu_parser.py @@ -1,3 +1,18 @@ +# Copyright (c) 2023, Huawei Technologies. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import sys import pandas as pd from collections import defaultdict diff --git a/profiler/performance_analyse/parser_helper.py b/profiler/performance_analyse/parser_helper.py index 6b91fdd5342c91e8ea251e0232e08bfc44012593..9cfab6910de020c7b2e6464fdb7b521b9e9c75e0 100644 --- a/profiler/performance_analyse/parser_helper.py +++ b/profiler/performance_analyse/parser_helper.py @@ -1,3 +1,18 @@ +# Copyright (c) 2023, Huawei Technologies. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json import os diff --git a/profiler/performance_analyse/profiling_parse.py b/profiler/performance_analyse/profiling_parse.py index 2f773495344e4f29a4d691d6818abf9b30e31a34..0e81f2d2199be54b135374847b4f3fe99cbe200c 100644 --- a/profiler/performance_analyse/profiling_parse.py +++ b/profiler/performance_analyse/profiling_parse.py @@ -1,3 +1,18 @@ +# Copyright (c) 2023, Huawei Technologies. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import argparse import os