From e93fe8a6006b7499adc5aac4c8c6228089e34c8f Mon Sep 17 00:00:00 2001 From: huxianglong Date: Tue, 24 Dec 2024 11:18:31 +0800 Subject: [PATCH 1/2] clean3 --- .../cluster_advice/cluster_pipeline_advice.py | 16 ++++++++-------- .../compute_advice/npu_fused/op_perf.py | 5 +++-- .../timeline_advice/op_schedule_advice.py | 5 +++-- .../timeline_advice/timeline_advice_base.py | 4 ++-- profiler/advisor/analyzer/analyzer_controller.py | 4 +++- .../communication_retransmission_analyzer.py | 2 +- .../ai_core_freq/ai_core_freq_checker.py | 6 ++---- .../advisor/analyzer/memory/memory_analyzer.py | 3 ++- .../schedule/fusion_ops/fusion_ops_analyzer.py | 3 ++- .../synchronize_stream_analyzer.py | 8 ++++---- profiler/advisor/common/async_analysis_status.py | 4 ++-- profiler/advisor/common/enum_params_parser.py | 4 ++-- profiler/advisor/common/graph/graph.py | 4 ++-- profiler/advisor/common/graph/graph_match.py | 4 ++-- profiler/advisor/common/graph/graph_parser.py | 4 ++-- profiler/advisor/common/profiling/ge_info.py | 4 ++-- profiler/advisor/common/profiling/msprof.py | 4 ++-- profiler/advisor/common/profiling/op_summary.py | 4 ++-- profiler/advisor/common/profiling/tasktime.py | 4 ++-- profiler/advisor/common/timeline/event.py | 4 ++-- .../advisor/common/timeline/fusion_ops_db.py | 4 ++-- profiler/advisor/common/version_control.py | 4 ++-- .../advisor/dataset/profiling/device_info.py | 13 +++++++++++++ .../dataset/profiling/profiling_dataset.py | 3 ++- .../timeline_op_collector.py | 13 +++++++++++++ .../display/prompt/en/dynamic_shape_prompt.py | 9 +++++---- profiler/advisor/utils/log.py | 12 ++++++++++++ profiler/advisor/utils/tools.py | 13 +++++++++++++ 28 files changed, 112 insertions(+), 55 deletions(-) diff --git a/profiler/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py b/profiler/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py index 1a057e2bde..5213d715e2 100644 --- a/profiler/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py +++ b/profiler/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py @@ -291,8 +291,8 @@ class ClusterPipelineAdvice(ClusterAdviceBase): process all rank profiling data by using multi-process """ start_time = time.time() - logger.info("Start to process %s rank profiling data with %s workers." - , str(len(self.rank_prof_dirs)), str(self.worker_num)) + logger.info("Start to process %s rank profiling data with %s workers.", + str(len(self.rank_prof_dirs)), str(self.worker_num)) with multiprocessing.Pool(self.worker_num) as pool: results = pool.map(self.work, self.rank_prof_dirs.items()) @@ -321,14 +321,14 @@ class ClusterPipelineAdvice(ClusterAdviceBase): timeslice_list = self.get_pipeline_timeslice(fine_data.hcom_ops, fine_data.hcom_tids, fine_data.min_ts, fine_data.max_ts) if not fine_data.fp_ops or not fine_data.bp_ops: - logger.info("[Rank %s] No frameWork data in trace view, only show stage and bubble." - , str(rank_id)) + logger.info("[Rank %s] No frameWork data in trace view, only show stage and bubble.", + str(rank_id)) elif len(fine_data.hcom_tids) > 1: - logger.warning("[Rank %s] More than one hcom tid found, only show stage and bubble." - , str(rank_id)) + logger.warning("[Rank %s] More than one hcom tid found, only show stage and bubble.", + str(rank_id)) else: - logger.info("[Rank %s] Found frameWork data in trace view, show fp bp and bubble." - , rank_id) + logger.info("[Rank %s] Found frameWork data in trace view, show fp bp and bubble.", + rank_id) bp_ops = self.get_fp_bp_bound_ops(fine_data) self.update_stage_fp_bp(timeslice_list, bp_ops) show_fp_bp = True diff --git a/profiler/advisor/advisor_backend/compute_advice/npu_fused/op_perf.py b/profiler/advisor/advisor_backend/compute_advice/npu_fused/op_perf.py index 7bcbed5a75..7fb2580f3e 100644 --- a/profiler/advisor/advisor_backend/compute_advice/npu_fused/op_perf.py +++ b/profiler/advisor/advisor_backend/compute_advice/npu_fused/op_perf.py @@ -151,11 +151,12 @@ class OpPerf: return (intput_size + output_size) / (Constant.BYTE_UNIT_TRANS * Constant.BYTE_UNIT_TRANS) def get_throughput(self): - # throughput(GB/s) + # throughput bytes (GB/s) if not self.task_duration or abs(self.task_duration) < 1e-6: print("[ERROR] There is no task_duration, do not assess vector op performance.") return 0 - return self.row[Constant.TITLE.SIZE] / Constant.BYTE_UNIT_TRANS / self.task_duration * Constant.UNIT_TRANS * Constant.UNIT_TRANS + return (self.row[Constant.TITLE.SIZE] / + Constant.BYTE_UNIT_TRANS / self.task_duration * Constant.UNIT_TRANS * Constant.UNIT_TRANS) def get_perf_color(self): return PerfColor.WHITE diff --git a/profiler/advisor/advisor_backend/timeline_advice/op_schedule_advice.py b/profiler/advisor/advisor_backend/timeline_advice/op_schedule_advice.py index 5bf2fed3cb..11008b2f37 100644 --- a/profiler/advisor/advisor_backend/timeline_advice/op_schedule_advice.py +++ b/profiler/advisor/advisor_backend/timeline_advice/op_schedule_advice.py @@ -66,8 +66,9 @@ class OpScheduleAdvice(TimelineAdviceBase): self.cur_bottleneck = f"NPU Utilication: {round(free_ratio * 100, 2)}%, " \ f"NPU Free Utilization: {round(cpt_ratio * 100, 2)}%." if len(self.preparse_data[self.PreParseType.SYNCHRONIZE]) > 1: - self.cur_advice = f"Device synchronize {len(self.preparse_data[self.PreParseType.SYNCHRONIZE])} times, " \ - "try to reduce synchronization statements to alleviate the bottleneck of operator delivery.\n" + self.cur_advice = \ + f"Device synchronize {len(self.preparse_data[self.PreParseType.SYNCHRONIZE])} times, " \ + "try to reduce synchronization statements to alleviate the bottleneck of operator delivery.\n" small_op_num = self.small_op_block(op_free, op_dur) small_op_ratio = small_op_num / len(op_dur) if op_dur else 0.0 if small_op_ratio > Constant.SMALL_OP_NUM_RATIO: diff --git a/profiler/advisor/advisor_backend/timeline_advice/timeline_advice_base.py b/profiler/advisor/advisor_backend/timeline_advice/timeline_advice_base.py index 1889972f84..98842cf84b 100644 --- a/profiler/advisor/advisor_backend/timeline_advice/timeline_advice_base.py +++ b/profiler/advisor/advisor_backend/timeline_advice/timeline_advice_base.py @@ -61,8 +61,8 @@ class TimelineAdviceBase(AdviceBase): (self.collection_path.endswith("ascend_pt") or self.collection_path.endswith("ascend_ms")): self.trace_view_path = os.path.join(self.collection_path, "ASCEND_PROFILER_OUTPUT", "trace_view.json") if not os.path.exists(self.trace_view_path): - logger.error("trace_view.json is not exist in the Path: %s." \ - , str(os.path.join(self.collection_path, "ASCEND_PROFILER_OUTPUT"))) + logger.error("trace_view.json is not exist in the Path: %s.", + str(os.path.join(self.collection_path, "ASCEND_PROFILER_OUTPUT"))) return False elif os.path.isfile(self.collection_path) and os.path.basename(self.collection_path) == "trace_view.json": self.trace_view_path = self.collection_path diff --git a/profiler/advisor/analyzer/analyzer_controller.py b/profiler/advisor/analyzer/analyzer_controller.py index 879b04ecb3..1a5a28b637 100644 --- a/profiler/advisor/analyzer/analyzer_controller.py +++ b/profiler/advisor/analyzer/analyzer_controller.py @@ -313,7 +313,9 @@ class AnalyzerController: >>> analyzer_controller = AnalyzerController() >>> analysis_kwargs = dict(advisor_analyze_processes=2, disable_profiling_comparison=True) >>> - >>> async_analysis_process = analyzer_controller.async_do_analysis(Interface.all_dimension, **analysis_kwargs) + >>> async_analysis_process = analyzer_controller.async_do_analysis( + >>> Interface.all_dimension, **analysis_kwargs) + >>> >>> >>> # query the job status every second >>> while True: diff --git a/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py b/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py index 31a024a644..bd5d76a40f 100644 --- a/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py +++ b/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py @@ -47,6 +47,6 @@ class RDMARetransmissionAnalyzer(BaseCommunicationAnalyzer): self.html = rdma_checker.make_render(self.html_render, add_render_list, priority=self.get_priority()) return self.result - def get_priority(self, max_mem_op_dur): + def get_priority(self, max_mem_op_dur=None): # 单次重传最少4s，高优先级 return PriorityBackgroundColor.high diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py index a0190b3c04..ef5f4ea22a 100644 --- a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py +++ b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py @@ -20,7 +20,6 @@ from profiler.advisor.display.prompt.base_prompt import BasePrompt from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.result.result import OptimizeResult from profiler.advisor.utils.utils import convert_to_float -from profiler.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() @@ -75,9 +74,8 @@ class AICoreFreqChecker: if self.decrease_freq_ops: # 按算子总耗时和降频比率 降序排列 - self.decrease_freq_ops.sort(key= - lambda x: (x[self.TOTAL_DURATION_INDEX], x[self.DECREASE_FREQ_RATIO_INDEX]), - reverse=True) + self.decrease_freq_ops.sort( + key=lambda x: (x[self.TOTAL_DURATION_INDEX], x[self.DECREASE_FREQ_RATIO_INDEX]), reverse=True) if not self.ai_core_freq_issues: return diff --git a/profiler/advisor/analyzer/memory/memory_analyzer.py b/profiler/advisor/analyzer/memory/memory_analyzer.py index 939e2de90c..799094ef55 100644 --- a/profiler/advisor/analyzer/memory/memory_analyzer.py +++ b/profiler/advisor/analyzer/memory/memory_analyzer.py @@ -39,7 +39,8 @@ class MemoryAnalyzer(BaseAnalyzer): memory_checker = MemoryOpsChecker() memory_checker.check_memory_ops(self.dataset) memory_checker.make_record(self.result) - memory_checker.make_render(self.html_render, priority=self.get_priority(memory_checker.max_mem_op_dur), rank=kwargs.get("rank")) + memory_checker.make_render( + self.html_render, priority=self.get_priority(memory_checker.max_mem_op_dur), rank=kwargs.get("rank")) return self.result def get_priority(self, max_mem_op_dur): diff --git a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py b/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py index 6e212e55b9..89e5807c4a 100644 --- a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py +++ b/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py @@ -210,7 +210,8 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): # by the regex index and then calculate the real index for matched fusion operators in event dataset for left, right in zip(total_ops_split_points, total_ops_split_points[1:]): matched_op_flag = True if (left, right) in matched_pattern_index_tuple else False - matched_ops_list = total_op_name[left: right].strip(Constant.OP_SEP).split(Constant.OP_SEP + Constant.OP_SEP) + matched_ops_list = \ + total_op_name[left: right].strip(Constant.OP_SEP).split(Constant.OP_SEP + Constant.OP_SEP) op_index.append([matched_op_flag, len(matched_ops_list)]) for i, _ in enumerate(op_index): if i > 0: diff --git a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py index 2f5ee5c47b..0dceaa1afd 100644 --- a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py +++ b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py @@ -15,10 +15,10 @@ import logging from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.result.result import OptimizeResult from profiler.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_checker import SynchronizeStreamChecker -from profiler.advisor.display.html.render import HTMLRender from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.result.result import OptimizeResult logger = logging.getLogger() @@ -43,5 +43,5 @@ class SynchronizeStreamAnalyzer(BaseAnalyzer): rank=kwargs.get("rank")) return self.result - def get_priority(self, max_mem_op_dur): - return max_mem_op_dur.priority + def get_priority(self, max_mem_op_dur): + return max_mem_op_dur.priority diff --git a/profiler/advisor/common/async_analysis_status.py b/profiler/advisor/common/async_analysis_status.py index 3d9a5d7c10..98bb458105 100644 --- a/profiler/advisor/common/async_analysis_status.py +++ b/profiler/advisor/common/async_analysis_status.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -""" + # Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" + class AsyncAnalysisStatus: diff --git a/profiler/advisor/common/enum_params_parser.py b/profiler/advisor/common/enum_params_parser.py index e9a3f6639c..2f0fe3a2a3 100644 --- a/profiler/advisor/common/enum_params_parser.py +++ b/profiler/advisor/common/enum_params_parser.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -""" + # Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" + import os import logging import typing diff --git a/profiler/advisor/common/graph/graph.py b/profiler/advisor/common/graph/graph.py index f86f5db7f2..ad68da9174 100644 --- a/profiler/advisor/common/graph/graph.py +++ b/profiler/advisor/common/graph/graph.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -""" + # Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" + import logging from typing import Dict, List, Tuple, Callable, Any, Optional, Union diff --git a/profiler/advisor/common/graph/graph_match.py b/profiler/advisor/common/graph/graph_match.py index fbf0a8abe8..1cf2fe170d 100644 --- a/profiler/advisor/common/graph/graph_match.py +++ b/profiler/advisor/common/graph/graph_match.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -""" + # Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" + import itertools import logging from functools import lru_cache diff --git a/profiler/advisor/common/graph/graph_parser.py b/profiler/advisor/common/graph/graph_parser.py index cffbfa19b4..bc0b04a357 100644 --- a/profiler/advisor/common/graph/graph_parser.py +++ b/profiler/advisor/common/graph/graph_parser.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -""" + # Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" + import os import logging import itertools diff --git a/profiler/advisor/common/profiling/ge_info.py b/profiler/advisor/common/profiling/ge_info.py index 91642f9679..927f5d3e71 100644 --- a/profiler/advisor/common/profiling/ge_info.py +++ b/profiler/advisor/common/profiling/ge_info.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -""" + # Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" + import logging import os from typing import Any, List diff --git a/profiler/advisor/common/profiling/msprof.py b/profiler/advisor/common/profiling/msprof.py index 54206bb471..76f51b672d 100644 --- a/profiler/advisor/common/profiling/msprof.py +++ b/profiler/advisor/common/profiling/msprof.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -""" + # Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" + import logging from typing import Dict, List diff --git a/profiler/advisor/common/profiling/op_summary.py b/profiler/advisor/common/profiling/op_summary.py index c042509df9..b1b416888f 100644 --- a/profiler/advisor/common/profiling/op_summary.py +++ b/profiler/advisor/common/profiling/op_summary.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -""" + # Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" + import logging from decimal import Decimal from typing import List, Any diff --git a/profiler/advisor/common/profiling/tasktime.py b/profiler/advisor/common/profiling/tasktime.py index 211800585a..7891f89039 100644 --- a/profiler/advisor/common/profiling/tasktime.py +++ b/profiler/advisor/common/profiling/tasktime.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -""" + # Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" + import logging from typing import Dict, List diff --git a/profiler/advisor/common/timeline/event.py b/profiler/advisor/common/timeline/event.py index 79ee63211c..f6afb73594 100644 --- a/profiler/advisor/common/timeline/event.py +++ b/profiler/advisor/common/timeline/event.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -""" + # Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" + from decimal import Decimal diff --git a/profiler/advisor/common/timeline/fusion_ops_db.py b/profiler/advisor/common/timeline/fusion_ops_db.py index 11727bf3ed..e2228f3480 100644 --- a/profiler/advisor/common/timeline/fusion_ops_db.py +++ b/profiler/advisor/common/timeline/fusion_ops_db.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -""" + # Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" + import logging import os diff --git a/profiler/advisor/common/version_control.py b/profiler/advisor/common/version_control.py index ec30b3be9d..f04f097ca4 100644 --- a/profiler/advisor/common/version_control.py +++ b/profiler/advisor/common/version_control.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -""" +# # Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" + import logging from typing import List diff --git a/profiler/advisor/dataset/profiling/device_info.py b/profiler/advisor/dataset/profiling/device_info.py index 707e726ce2..52c811e6dd 100644 --- a/profiler/advisor/dataset/profiling/device_info.py +++ b/profiler/advisor/dataset/profiling/device_info.py @@ -1,3 +1,16 @@ +# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ profiling info """ diff --git a/profiler/advisor/dataset/profiling/profiling_dataset.py b/profiler/advisor/dataset/profiling/profiling_dataset.py index cd1d953aa0..99f11e171a 100644 --- a/profiler/advisor/dataset/profiling/profiling_dataset.py +++ b/profiler/advisor/dataset/profiling/profiling_dataset.py @@ -37,7 +37,8 @@ class ProfilingDataset(Dataset): def __init__(self, collection_path, data: dict, **kwargs) -> None: self.cann_version = kwargs.get(Constant.CANN_VERSION, EnumParamsParser().get_default(Constant.CANN_VERSION)) - self.prof_type = kwargs.get(Constant.PROFILING_TYPE_UNDER_LINE, EnumParamsParser().get_default(Constant.PROFILING_TYPE_UNDER_LINE)) + self.prof_type = kwargs.get( + Constant.PROFILING_TYPE_UNDER_LINE, EnumParamsParser().get_default(Constant.PROFILING_TYPE_UNDER_LINE)) self.patterns = self.parse_pattern() self.current_version_pattern = self.get_current_version_pattern() self._info = None diff --git a/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py b/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py index ab3433a754..a1aa68d208 100644 --- a/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py +++ b/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py @@ -1,3 +1,16 @@ +# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging import math import os diff --git a/profiler/advisor/display/prompt/en/dynamic_shape_prompt.py b/profiler/advisor/display/prompt/en/dynamic_shape_prompt.py index b350f603f1..e8a785b2f5 100644 --- a/profiler/advisor/display/prompt/en/dynamic_shape_prompt.py +++ b/profiler/advisor/display/prompt/en/dynamic_shape_prompt.py @@ -17,8 +17,9 @@ class DynamicShapePrompt(object): RANK_ID = "RANK {} " PROBLEM = "Dynamic Shape Operator" DESCRIPTION = "Found all operators are dynamic shape" - ENABLE_COMPILED_SUGGESTION = "1. Please try to set environment by execute `export HOST_CACHE_CAPACITY=20`.\n." \ - "2. Please place the following code at the entrance of the python script to disable jit compile.\n " \ - "Code: `torch_npu.npu.set_compile_mode(jit_compile=False) \n " \ - "torch_npu.npu.config.allow_internal_format = False`.\n" + ENABLE_COMPILED_SUGGESTION = \ + "1. Please try to set environment by execute `export HOST_CACHE_CAPACITY=20`.\n." \ + "2. Please place the following code at the entrance of the python script to disable jit compile.\n " \ + "Code: `torch_npu.npu.set_compile_mode(jit_compile=False) \n " \ + "torch_npu.npu.config.allow_internal_format = False`.\n" RELEASE_SUGGESTION = "for details please refer to link : LINK" diff --git a/profiler/advisor/utils/log.py b/profiler/advisor/utils/log.py index 5d9757ebf4..853d9e1559 100644 --- a/profiler/advisor/utils/log.py +++ b/profiler/advisor/utils/log.py @@ -1,3 +1,15 @@ +# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ log module """ diff --git a/profiler/advisor/utils/tools.py b/profiler/advisor/utils/tools.py index 90c2c9cf0b..4887dc4958 100644 --- a/profiler/advisor/utils/tools.py +++ b/profiler/advisor/utils/tools.py @@ -1,3 +1,16 @@ +# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from functools import partial import click -- Gitee From 722e5269001f1d3a49ee40baef2d5456403c43e8 Mon Sep 17 00:00:00 2001 From: huxianglong Date: Tue, 24 Dec 2024 11:29:49 +0800 Subject: [PATCH 2/2] clean --- .../overall_advice/overall_summary_advice.py | 6 ++---- .../computation/ai_core_freq/ai_core_freq_checker.py | 1 + profiler/advisor/dataset/timeline_event_dataset.py | 8 ++++---- profiler/advisor/utils/utils.py | 8 ++++---- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/profiler/advisor/advisor_backend/overall_advice/overall_summary_advice.py b/profiler/advisor/advisor_backend/overall_advice/overall_summary_advice.py index effa9099e2..4310c41ce1 100644 --- a/profiler/advisor/advisor_backend/overall_advice/overall_summary_advice.py +++ b/profiler/advisor/advisor_backend/overall_advice/overall_summary_advice.py @@ -14,14 +14,12 @@ # limitations under the License. import os -from advisor_backend.advice_base import AdviceBase - from profiler.advisor.display.prompt.base_prompt import BasePrompt +from profiler.prof_common.additional_args_manager import AdditionalArgsManager from profiler.prof_common.constant import Constant from compare_interface.comparison_interface import ComparisonInterface - -from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from advisor_backend.advice_base import AdviceBase class OverallSummaryAdvice(AdviceBase): diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py index ef5f4ea22a..b203514208 100644 --- a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py +++ b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py @@ -20,6 +20,7 @@ from profiler.advisor.display.prompt.base_prompt import BasePrompt from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.result.result import OptimizeResult from profiler.advisor.utils.utils import convert_to_float +from profiler.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() diff --git a/profiler/advisor/dataset/timeline_event_dataset.py b/profiler/advisor/dataset/timeline_event_dataset.py index fbac2c5bf0..77d64267c4 100644 --- a/profiler/advisor/dataset/timeline_event_dataset.py +++ b/profiler/advisor/dataset/timeline_event_dataset.py @@ -80,10 +80,10 @@ class BaseTimelineEventDataset: kwargs = {} if func_name == FrequencyCollector.__name__: ops_with_task_type = getattr(self, "ops_with_task_type", {}).values() - kwargs["ai_core_ops"] = [ - op for op in ops_with_task_type if - op.get(Constant.TASK_TYPE) in [Constant.AI_CORE, Constant.MIX_AIC] - ] + kwargs["ai_core_ops"] = [op + for op in ops_with_task_type if + op.get(Constant.TASK_TYPE) in [Constant.AI_CORE, Constant.MIX_AIC] + ] return kwargs def add_event(self, index, event): diff --git a/profiler/advisor/utils/utils.py b/profiler/advisor/utils/utils.py index 710084ef8c..0e928edde9 100644 --- a/profiler/advisor/utils/utils.py +++ b/profiler/advisor/utils/utils.py @@ -119,10 +119,10 @@ def singleton(cls): members = inspect.getmembers(base_class) # 过滤出函数对象 - function_objs = [ - member[1] for member in members - if inspect.isfunction(member[1]) or inspect.ismethod(member[1]) - ] + function_objs = [member[1] + for member in members + if inspect.isfunction(member[1]) or inspect.ismethod(member[1]) + ] for function_obj in function_objs: if inspect.isfunction(function_obj) and not is_static_func(function_obj): continue -- Gitee