From 6b9577b0b384e414d972540cbe4f1958d208f73b Mon Sep 17 00:00:00 2001 From: stby <295887736@qq.com> Date: Tue, 19 Nov 2024 21:10:38 +0800 Subject: [PATCH 1/2] feature: refactor constant and file manager --- .../cluster_advice/cluster_advice_base.py | 2 +- .../cluster_advice/cluster_pipeline_advice.py | 2 +- .../cluster_advice/kernel_cluster_advice.py | 6 +- .../cluster_advice/slow_link_advice.py | 2 +- .../cluster_advice/slow_rank_advice.py | 2 +- .../common_func_advisor/trace_view_json.py | 2 +- .../compute_advice/compute_advice_base.py | 2 +- .../overall_advice/overall_summary_advice.py | 3 +- .../timeline_advice/timeline_advice_base.py | 2 +- .../advisor/analyzer/analyzer_controller.py | 17 +- profiler/advisor/analyzer/base_analyzer.py | 8 +- .../Communication_retransmission_checker.py | 2 +- .../analyzer/cluster/slow_link_analyzer.py | 6 +- .../analyzer/cluster/slow_rank_analyzer.py | 10 +- .../alignment/byte_alignment_checker.py | 6 +- .../bandwidth_contention_checker.py | 2 +- .../communication/packet/packet_checker.py | 2 +- .../communication_retransmission_checker.py | 6 +- .../analyzer/comparison/comparison_checker.py | 6 +- .../computation/aicpu/aicpu_checker.py | 15 +- .../computation/bound/block_dim_checker.py | 4 +- .../bound/operator_bound_checker.py | 4 +- .../op_compile/dynamic_shape_checker.py | 1 - .../analyzer/computation/operator_checker.py | 18 +- .../pp_stage_computation_analyzer.py | 1 - .../analyzer/dataloader/dataloader_checker.py | 2 +- .../advisor/analyzer/memory/memory_checker.py | 2 +- .../overall/environment_variable_checker.py | 2 +- .../overall/overall_summary_analyzer.py | 3 +- .../dispatch/timeline_op_dispatch_analyzer.py | 6 +- .../fusion_ops/fusion_ops_analyzer.py | 36 +- .../fusion_ops/timeline_api_stack_checker.py | 22 +- .../analyzer/schedule/gc/gc_checker.py | 8 +- .../schedule/syncbn/syncbn_checker.py | 2 +- .../synchronize_stream_checker.py | 10 +- .../schedule/timeline_base_checker.py | 14 +- profiler/advisor/common/constant.py | 161 -------- profiler/advisor/common/enum_params_parser.py | 2 +- profiler/advisor/common/graph/graph_parser.py | 2 +- .../advisor/common/timeline/fusion_ops_db.py | 38 +- .../timeline/fusion_ops_rule_handler.py | 4 +- .../ai_core_freq/ai_core_freq_dataset.py | 1 - .../dataset/cluster/cluster_dataset.py | 27 +- .../communication/communication_dataset.py | 6 +- .../dataset/environment_variable_dataset.py | 6 +- .../advisor/dataset/profiling/device_info.py | 2 +- .../dataset/profiling/profiling_dataset.py | 10 +- .../dataset/profiling/profiling_parser.py | 2 +- .../advisor/dataset/timeline_event_dataset.py | 6 +- .../timeline_op_collector.py | 34 +- profiler/advisor/display/html/render.py | 4 +- profiler/advisor/result/result.py | 6 +- profiler/advisor/utils/file.py | 6 +- profiler/advisor/utils/log.py | 6 +- profiler/advisor/utils/utils.py | 8 +- profiler/cli/analyze_cli.py | 38 +- .../analysis/analysis_facade.py | 3 +- .../cluster_analyse/analysis/base_analysis.py | 4 +- .../analysis/comm_matrix_analysis.py | 2 +- .../analysis/communication_analysis.py | 2 +- .../analysis/host_info_analysis.py | 2 +- .../analysis/step_trace_time_analysis.py | 4 +- profiler/cluster_analyse/cluster_analysis.py | 6 +- .../pytorch_data_preprocessor.py | 4 +- .../cluster_utils/data_transfer_adapter.py | 2 +- .../cluster_analyse/common_func/constant.py | 121 ------ .../cluster_analyse/common_func/db_manager.py | 4 +- .../base_communication_group.py | 2 +- .../communication_db_group.py | 2 +- .../communication_group_generator.py | 2 +- .../communication_json_group.py | 2 +- .../comparator/api_compare_comparator.py | 2 +- .../comparator/communication_comparator.py | 2 +- .../comparator/kernel_compare_comparator.py | 2 +- .../comparator/module_comparetor.py | 2 +- .../comparator/overall_metrics_comparator.py | 2 +- .../overall_performance_comparator.py | 2 +- .../compare_bean/api_compare_bean.py | 2 +- .../compare_bean/communication_bean.py | 2 +- .../compare_bean/kernel_compare_bean.py | 2 +- .../compare_bean/memory_compare_bean.py | 2 +- .../compare_bean/memory_statistic_bean.py | 2 +- .../compare_bean/module_compare_bean.py | 2 +- .../compare_bean/module_statistic_bean.py | 2 +- .../compare_bean/operator_compare_bean.py | 2 +- .../compare_bean/operator_statistic_bean.py | 2 +- .../origin_data_bean/compare_event.py | 2 +- .../origin_data_bean/kernel_details_bean.py | 2 +- .../origin_data_bean/trace_event_bean.py | 2 +- .../compare_bean/overall_metrics_bean.py | 2 +- .../compare_bean/profiling_info.py | 2 +- .../compare_backend/comparison_generator.py | 2 +- .../data_prepare/module_data_prepare.py | 2 +- .../data_prepare/operator_data_prepare.py | 2 +- .../data_prepare/sequence_pre_matching.py | 2 +- .../disaggregate/overall_perf_interface.py | 2 +- .../generator/detail_performance_generator.py | 2 +- .../interface/compare_interface.py | 3 +- .../interface/overall_interface.py | 2 +- .../profiling_parser/base_profiling_parser.py | 6 +- .../profiling_parser/gpu_profiling_parser.py | 2 +- .../profiling_parser/npu_profiling_parser.py | 14 +- .../compare_backend/utils/args_manager.py | 7 +- .../compare_backend/utils/constant.py | 116 ------ .../compare_backend/utils/excel_config.py | 2 +- .../compare_backend/utils/file_reader.py | 85 ----- .../compare_backend/utils/torch_op_node.py | 2 +- .../compare_backend/view/excel_view.py | 2 +- .../compare_interface/comparison_interface.py | 2 +- profiler/prof_common/constant.py | 349 ++++++++++++++++++ .../file_manager.py | 63 ++-- profiler/test/run_st.py | 2 +- profiler/test/run_ut.py | 2 +- .../test_cluster_pipeline_advice.py | 4 +- .../test_kernel_cluster_advice.py | 2 +- .../test_cluster_step_trace_time_bean.py | 1 - .../test_synchronize_stream.py | 10 +- .../test_timeline_op_compile_checker.py | 4 +- .../common_func/test_file_manager.py | 2 +- .../test_npu_profiling_parser.py | 2 +- .../compare_tools/utils/test_file_reader.py | 19 - 121 files changed, 689 insertions(+), 835 deletions(-) delete mode 100644 profiler/advisor/common/constant.py delete mode 100644 profiler/cluster_analyse/common_func/constant.py delete mode 100644 profiler/compare_tools/compare_backend/utils/constant.py delete mode 100644 profiler/compare_tools/compare_backend/utils/file_reader.py rename profiler/{cluster_analyse/common_func => prof_common}/file_manager.py (78%) delete mode 100644 profiler/test/ut/compare_tools/utils/test_file_reader.py diff --git a/profiler/advisor/advisor_backend/cluster_advice/cluster_advice_base.py b/profiler/advisor/advisor_backend/cluster_advice/cluster_advice_base.py index 3a14b43054b..60e5a75482a 100644 --- a/profiler/advisor/advisor_backend/cluster_advice/cluster_advice_base.py +++ b/profiler/advisor/advisor_backend/cluster_advice/cluster_advice_base.py @@ -16,10 +16,10 @@ import os from abc import abstractmethod -from common_func.constant import Constant from advice_base import AdviceBase from cluster_analysis import Interface from profiler.advisor.advisor_backend.logger import Logger +from profiler.prof_common.constant import Constant logger = Logger() diff --git a/profiler/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py b/profiler/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py index ea52042886c..47fd73d0ab2 100644 --- a/profiler/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py +++ b/profiler/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py @@ -26,13 +26,13 @@ from collections import deque from decimal import Decimal from dataclasses import dataclass -from common_func.file_manager import FileManager from common_func_advisor.constant import Constant from common_func_advisor.trace_view_preprocessor import FineTraceViewData from common_func_advisor.trace_view_preprocessor import TraceViewPreProcessor from cluster_advice.cluster_advice_base import ClusterAdviceBase from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor from profiler.advisor.advisor_backend.logger import Logger +from profiler.prof_common.file_manager import FileManager logger = Logger() diff --git a/profiler/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py b/profiler/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py index 474e5d99e25..89a9ccf7727 100644 --- a/profiler/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py +++ b/profiler/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py @@ -15,12 +15,12 @@ import os import pandas as pd -from common_func.path_manager import PathManager -from common_func.constant import Constant from common_func_advisor.constant import Constant as AdvisorConstant from cluster_advice.cluster_advice_base import ClusterAdviceBase from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager +from profiler.prof_common.constant import Constant +from profiler.prof_common.path_manager import PathManager class KernelClusterAdvice(ClusterAdviceBase): diff --git a/profiler/advisor/advisor_backend/cluster_advice/slow_link_advice.py b/profiler/advisor/advisor_backend/cluster_advice/slow_link_advice.py index 6ade57ccf9c..065e9a9ca24 100644 --- a/profiler/advisor/advisor_backend/cluster_advice/slow_link_advice.py +++ b/profiler/advisor/advisor_backend/cluster_advice/slow_link_advice.py @@ -16,8 +16,8 @@ import os from collections import defaultdict from common_func_advisor.constant import Constant -from common_func.file_manager import FileManager from cluster_advice.cluster_advice_base import ClusterAdviceBase +from profiler.prof_common.file_manager import FileManager class SlowLinkAdvice(ClusterAdviceBase): diff --git a/profiler/advisor/advisor_backend/cluster_advice/slow_rank_advice.py b/profiler/advisor/advisor_backend/cluster_advice/slow_rank_advice.py index 4e789fb7fb6..c8a067d0ee5 100644 --- a/profiler/advisor/advisor_backend/cluster_advice/slow_rank_advice.py +++ b/profiler/advisor/advisor_backend/cluster_advice/slow_rank_advice.py @@ -16,9 +16,9 @@ import os from collections import defaultdict from common_func_advisor.constant import Constant -from common_func.file_manager import FileManager from cluster_advice.cluster_advice_base import ClusterAdviceBase from prof_bean_advisor.cluster_step_trace_time_bean import ClusterStepTraceTimeBean +from profiler.prof_common.file_manager import FileManager class SlowRankAdvice(ClusterAdviceBase): diff --git a/profiler/advisor/advisor_backend/common_func_advisor/trace_view_json.py b/profiler/advisor/advisor_backend/common_func_advisor/trace_view_json.py index f77140db697..a6337cd5798 100644 --- a/profiler/advisor/advisor_backend/common_func_advisor/trace_view_json.py +++ b/profiler/advisor/advisor_backend/common_func_advisor/trace_view_json.py @@ -23,7 +23,7 @@ from typing import List import pandas as pd -from common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager from profiler.advisor.advisor_backend.logger import Logger logger = Logger() diff --git a/profiler/advisor/advisor_backend/compute_advice/compute_advice_base.py b/profiler/advisor/advisor_backend/compute_advice/compute_advice_base.py index cafbafd8e28..811cce84e0d 100644 --- a/profiler/advisor/advisor_backend/compute_advice/compute_advice_base.py +++ b/profiler/advisor/advisor_backend/compute_advice/compute_advice_base.py @@ -18,7 +18,7 @@ from collections import defaultdict import os from advice_base import AdviceBase -from common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager class ComputeAdviceBase(AdviceBase): diff --git a/profiler/advisor/advisor_backend/overall_advice/overall_summary_advice.py b/profiler/advisor/advisor_backend/overall_advice/overall_summary_advice.py index f5bfc351f28..80a83e58698 100644 --- a/profiler/advisor/advisor_backend/overall_advice/overall_summary_advice.py +++ b/profiler/advisor/advisor_backend/overall_advice/overall_summary_advice.py @@ -15,7 +15,8 @@ import os from advisor_backend.advice_base import AdviceBase -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant + from compare_interface.comparison_interface import ComparisonInterface diff --git a/profiler/advisor/advisor_backend/timeline_advice/timeline_advice_base.py b/profiler/advisor/advisor_backend/timeline_advice/timeline_advice_base.py index 2b8f470a147..e2aa1e5ae12 100644 --- a/profiler/advisor/advisor_backend/timeline_advice/timeline_advice_base.py +++ b/profiler/advisor/advisor_backend/timeline_advice/timeline_advice_base.py @@ -20,7 +20,7 @@ import os import logging from advice_base import AdviceBase -from common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/profiler/advisor/analyzer/analyzer_controller.py b/profiler/advisor/analyzer/analyzer_controller.py index cde7e2d413b..4da80e08f46 100644 --- a/profiler/advisor/analyzer/analyzer_controller.py +++ b/profiler/advisor/analyzer/analyzer_controller.py @@ -33,7 +33,6 @@ from profiler.advisor.analyzer.cluster.slow_link_analyzer import SlowLinkAnalyze from profiler.advisor.analyzer.computation.pp_stage_computation_analyzer import PPStageComputationAnalyzer from profiler.advisor.analyzer.overall.overall_summary_analyzer import OverallSummaryAnalyzer from profiler.advisor.config.config import Config -from profiler.advisor.common import constant as const from profiler.advisor.common.analyzer_scopes import SupportedScopes from profiler.advisor.common.async_analysis_status import AsyncAnalysisStatus from profiler.advisor.common.enum_params_parser import EnumParamsParser @@ -41,7 +40,7 @@ from profiler.advisor.utils.utils import Timer, safe_index_value, safe_division, from profiler.advisor.interface.interface import Interface from profiler.cluster_analyse.cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor from profiler.prof_common.path_manager import PathManager -from profiler.compare_tools.compare_backend.utils.constant import Constant as CompareConstant +from profiler.prof_common.constant import Constant # 以spawn模式启动多进程，避免fork主进程资源。如果主进程逻辑较为复杂，fork可能会导致异常。 mp.set_start_method("spawn", force=True) @@ -397,9 +396,9 @@ class AnalyzerController: # kernel/api 比对 compare_profiling_list = [ dict(profiling_path=profiling_path, benchmark_profiling_path=benchmark_profiling_path, - compare_mode=CompareConstant.KERNEL_COMPARE), + compare_mode=Constant.KERNEL_COMPARE), dict(profiling_path=profiling_path, benchmark_profiling_path=benchmark_profiling_path, - compare_mode=CompareConstant.API_COMPARE) + compare_mode=Constant.API_COMPARE) ] job_list += self._profiling_comparison(compare_profiling_list) @@ -545,7 +544,7 @@ class AnalyzerController: benchmark_profiling_path=self._get_profiling_path_by_rank(profiling_path, fast_rank_id), step=slow_step, benchmark_step=fast_step, rank=slow_rank_id, benchmark_rank=fast_rank_id, - compare_mode=CompareConstant.API_COMPARE, + compare_mode=Constant.API_COMPARE, step_duration=self.slow_rank_analyzer.get_step_duration(slow_rank_id, slow_step)) job_list += self.schedule_analysis(**kwargs) @@ -736,7 +735,7 @@ class AnalyzerController: def _profiling_comparison(self, compare_profiling_list): job_list = [] - disable_profiling_comparison = os.getenv(const.DISABLE_PROFILING_COMPARISON) + disable_profiling_comparison = os.getenv(Constant.DISABLE_PROFILING_COMPARISON) if disable_profiling_comparison is not None and disable_profiling_comparison.lower() == "true": logger.info( "Skip profiling comparison due to longer processing time due to env 'DISABLE_PROFILING_COMPARISON'") @@ -787,7 +786,7 @@ class AnalyzerController: if isinstance(target_cluster_analyzer, SlowRankAnalyzer): comparison_dims = [SlowRankAnalyzer.COMPUTE, SlowRankAnalyzer.FREE] - comparison_modes = [CompareConstant.KERNEL_COMPARE, CompareConstant.API_COMPARE] + comparison_modes = [Constant.KERNEL_COMPARE, Constant.API_COMPARE] elif isinstance(target_cluster_analyzer, SlowLinkAnalyzer): comparison_dims = [SlowLinkAnalyzer.SDMA_BANDWIDTH, SlowLinkAnalyzer.RDMA_BANDWIDTH] comparison_modes = [None, None] @@ -913,7 +912,7 @@ class AnalyzerController: benchmark_step=benchmark_step, profiling_path=self._get_profiling_path_by_rank(profiling_path, rank_id), benchmark_profiling_path=self._get_profiling_path_by_rank(profiling_path, benchmark_rank_id), - compare_mode=CompareConstant.KERNEL_COMPARE, + compare_mode=Constant.KERNEL_COMPARE, step_duration=self.slow_rank_analyzer.get_step_duration(rank_id, step) ) ) @@ -953,7 +952,7 @@ class AnalyzerController: kwargs = dict(profiling_path=self._get_profiling_path_by_rank(profiling_path, slow_rank_id), benchmark_profiling_path=self._get_profiling_path_by_rank(profiling_path, fast_rank_id), step=slow_step, benchmark_step=fast_step, rank=slow_rank_id, benchmark_rank=fast_rank_id, - compare_mode=CompareConstant.KERNEL_COMPARE, + compare_mode=Constant.KERNEL_COMPARE, step_duration=self.slow_rank_analyzer.get_step_duration(slow_rank_id, slow_step)) job_list += self.computation_analysis(**kwargs) diff --git a/profiler/advisor/analyzer/base_analyzer.py b/profiler/advisor/analyzer/base_analyzer.py index 38b7ea0be68..b7fd46df44d 100644 --- a/profiler/advisor/analyzer/base_analyzer.py +++ b/profiler/advisor/analyzer/base_analyzer.py @@ -17,7 +17,7 @@ from functools import wraps from typing import Dict, List, Union from abc import abstractmethod, ABCMeta -from profiler.advisor.common import constant +from profiler.prof_common.constant import Constant from profiler.advisor.common.enum_params_parser import EnumParamsParser from profiler.advisor.common.version_control import VersionControl from profiler.advisor.dataset.dataset import Dataset @@ -30,7 +30,7 @@ logger = logging.getLogger() class BaseAnalyzer(VersionControl, metaclass=ABCMeta): - _SUPPORT_VERSIONS = EnumParamsParser().get_options(constant.CANN_VERSION) + _SUPPORT_VERSIONS = EnumParamsParser().get_options(Constant.CANN_VERSION) ANALYZER_HIGH_PRIORITY_TIME_RATIO = 0.05 ANALYZER_MEDIUM_PRIORITY_TIME_RATIO = 0.03 @@ -38,8 +38,8 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): def __init__(self, collection_path, n_processes: int = 1, **kwargs): self.n_processes = n_processes - self.cann_version = kwargs.get(constant.CANN_VERSION, EnumParamsParser().get_default(constant.CANN_VERSION)) - self.torch_version = kwargs.get(constant.TORCH_VERSION, EnumParamsParser().get_default(constant.TORCH_VERSION)) + self.cann_version = kwargs.get(Constant.CANN_VERSION, EnumParamsParser().get_default(Constant.CANN_VERSION)) + self.torch_version = kwargs.get(Constant.TORCH_VERSION, EnumParamsParser().get_default(Constant.TORCH_VERSION)) self.html_render = HTMLRender() self.collection_path = collection_path self.kwargs = kwargs diff --git a/profiler/advisor/analyzer/cluster/Communication_retransmission_checker.py b/profiler/advisor/analyzer/cluster/Communication_retransmission_checker.py index c63fc12f27a..e3c6fa33222 100644 --- a/profiler/advisor/analyzer/cluster/Communication_retransmission_checker.py +++ b/profiler/advisor/analyzer/cluster/Communication_retransmission_checker.py @@ -19,7 +19,7 @@ from collections import defaultdict from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/cluster/slow_link_analyzer.py b/profiler/advisor/analyzer/cluster/slow_link_analyzer.py index 259e5eb0c42..10459e71929 100644 --- a/profiler/advisor/analyzer/cluster/slow_link_analyzer.py +++ b/profiler/advisor/analyzer/cluster/slow_link_analyzer.py @@ -18,7 +18,7 @@ from typing import Dict, List import logging from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.common import constant +from profiler.prof_common.constant import Constant from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset @@ -105,7 +105,7 @@ class SlowLinkAnalyzer(BaseAnalyzer): data_list = [] for step_rank, rank_bw in self.rank_bw_dict.items(): - step_rank_list = list(map(convert_to_int, step_rank.split(constant.STEP_RANK_SEP))) + step_rank_list = list(map(convert_to_int, step_rank.split(Constant.STEP_RANK_SEP))) value_list = [rank_bw.get(i, 0) for i in headers] data_list.append(step_rank_list + value_list) data_list.sort(key=lambda x: (x[0], x[1])) # 按rank_id排序 @@ -181,7 +181,7 @@ class SlowLinkAnalyzer(BaseAnalyzer): min_bandwidth_rank_id = self.format_datas.get("data")[min_bandwidth_index][rank_id_index] if step_index is None: - max_bandwidth_step, min_bandwidth_step = constant.DEFAULT_STEP, constant.DEFAULT_STEP + max_bandwidth_step, min_bandwidth_step = Constant.DEFAULT_STEP, Constant.DEFAULT_STEP else: max_bandwidth_step = self.format_datas.get("data")[max_bandwidth_index][step_index] min_bandwidth_step = self.format_datas.get("data")[min_bandwidth_index][step_index] diff --git a/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py b/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py index 165dec7db7f..0d73a74eea4 100644 --- a/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py +++ b/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py @@ -16,7 +16,7 @@ import logging from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.common import constant +from profiler.prof_common.constant import Constant from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.dataset.cluster.cluster_dataset import ClusterStepTraceTimeDataset @@ -65,7 +65,7 @@ class SlowRankAnalyzer(BaseAnalyzer): logger.error( "Slow rank analysis failed, " "please ensure file 'step_trace_time.csv' exists in your profiling directory %s", - constant.ASCEND_PROFILER_OUTPUT) + Constant.ASCEND_PROFILER_OUTPUT) return self.result self.process() self.make_record() @@ -113,7 +113,7 @@ class SlowRankAnalyzer(BaseAnalyzer): headers = ["step", "rank_id", "compute(us)", "communication(us)", "free(us)"] data_list = [] for key, value in self.step_trace_dict.items(): - step, rank_id = key.split(constant.STEP_RANK_SEP) + step, rank_id = key.split(Constant.STEP_RANK_SEP) data_list.append([convert_to_int(step), convert_to_int(rank_id)] + value) if step and step not in self._steps: self._steps.add(step) @@ -167,7 +167,7 @@ class SlowRankAnalyzer(BaseAnalyzer): max_time_step = self.format_datas.get("data")[max_time_index][step_index] min_time_step = self.format_datas.get("data")[min_time_index][step_index] else: - max_time_step, min_time_step = constant.DEFAULT_STEP, constant.DEFAULT_STEP + max_time_step, min_time_step = Constant.DEFAULT_STEP, Constant.DEFAULT_STEP global_step_rank["maximum"] = {"rank_id": max_time_rank_id, "step": max_time_step} global_step_rank["minimum"] = {"rank_id": min_time_rank_id, "step": min_time_step} @@ -192,7 +192,7 @@ class SlowRankAnalyzer(BaseAnalyzer): if step_index is not None: step_list = [tuple_list[step_index] for tuple_list in self.format_datas.get("data")] else: - step_list = [constant.DEFAULT_STEP] * len(rank_list) + step_list = [Constant.DEFAULT_STEP] * len(rank_list) for index, stage in enumerate(self.stages): tmp_step_list, tmp_rank_list, tmp_time_list = [], [], [] diff --git a/profiler/advisor/analyzer/communication/alignment/byte_alignment_checker.py b/profiler/advisor/analyzer/communication/alignment/byte_alignment_checker.py index f06389148c7..3a9ebd86eca 100644 --- a/profiler/advisor/analyzer/communication/alignment/byte_alignment_checker.py +++ b/profiler/advisor/analyzer/communication/alignment/byte_alignment_checker.py @@ -15,14 +15,14 @@ import logging import os from typing import List -from profiler.advisor.common.constant import COMMUNICATION_B_TO_GB, US_TO_S from profiler.advisor.dataset.communication.hccl_detail_dataset import HcclDetailDataset from profiler.advisor.dataset.profiling.info_collection import HcclTask from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager from profiler.advisor.utils.utils import safe_division +from profiler.prof_common.constant import Constant logger = logging.getLogger() @@ -58,7 +58,7 @@ class ByteAlignmentChecker: if abs(duration) < 1e-15: bandwidth = 0 else: - bandwidth = (size * COMMUNICATION_B_TO_GB) / (duration * US_TO_S) + bandwidth = (size * Constant.COMMUNICATION_B_TO_GB) / (duration * Constant.US_TO_S) return round(bandwidth, 4) def check_alignment(self, hccl_detail_dataset: HcclDetailDataset) -> None: diff --git a/profiler/advisor/analyzer/communication/contention/bandwidth_contention_checker.py b/profiler/advisor/analyzer/communication/contention/bandwidth_contention_checker.py index cce7aa85401..acba63dc182 100644 --- a/profiler/advisor/analyzer/communication/contention/bandwidth_contention_checker.py +++ b/profiler/advisor/analyzer/communication/contention/bandwidth_contention_checker.py @@ -19,7 +19,7 @@ from profiler.advisor.dataset.communication.communication_dataset import Communi from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager from profiler.advisor.utils.utils import convert_to_float from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo from profiler.advisor.dataset.profiling.info_collection import OpInfo diff --git a/profiler/advisor/analyzer/communication/packet/packet_checker.py b/profiler/advisor/analyzer/communication/packet/packet_checker.py index 6ddf17c43fd..250d76af30c 100644 --- a/profiler/advisor/analyzer/communication/packet/packet_checker.py +++ b/profiler/advisor/analyzer/communication/packet/packet_checker.py @@ -17,7 +17,7 @@ import os from profiler.advisor.dataset.communication.communication_dataset import CommunicationDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager from profiler.advisor.utils.utils import convert_to_float logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py b/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py index 1720633b36f..bde96791865 100644 --- a/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py +++ b/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py @@ -16,12 +16,12 @@ import logging import os from typing import Dict, List from collections import defaultdict -from profiler.advisor.common.constant import TOTAL_OP_INFO from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo +from profiler.prof_common.constant import Constant logger = logging.getLogger() @@ -75,7 +75,7 @@ class CommunicationRetransmissionChecker: """ for group_name, hccl_group_dict in hccl_dataset.hccl_dict.items(): for op_name, hccl_op_dict in hccl_group_dict.items(): - if op_name == TOTAL_OP_INFO: + if op_name == Constant.TOTAL_OP_INFO: continue for step_id, hccl_list in hccl_op_dict.items(): if self.step_id and step_id != self.step_id: # 传输指定step（self.step_id）情况下，非目标step跳过 diff --git a/profiler/advisor/analyzer/comparison/comparison_checker.py b/profiler/advisor/analyzer/comparison/comparison_checker.py index ad4cb83d33c..dc12f9353ba 100644 --- a/profiler/advisor/analyzer/comparison/comparison_checker.py +++ b/profiler/advisor/analyzer/comparison/comparison_checker.py @@ -17,8 +17,8 @@ import logging from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.utils.utils import safe_index_value, convert_to_float, convert_to_int -from profiler.compare_tools.compare_backend.utils.constant import Constant as CompareConstant from profiler.compare_tools.compare_interface.comparison_interface import ComparisonInterface +from profiler.prof_common.constant import Constant logger = logging.getLogger() @@ -28,8 +28,8 @@ class ComparisonChecker: SHOW_TOPK = 10 DIFF_AVG_RATIO = "Diff Avg Ratio" COMPARE_MODE_TO_DESC = { - CompareConstant.KERNEL_COMPARE: "Kernel compare", - CompareConstant.API_COMPARE: "Api compare", + Constant.KERNEL_COMPARE: "Kernel compare", + Constant.API_COMPARE: "Api compare", } def __init__(self, profiling_path, benchmark_profiling_path, step=None, benchmark_step=None, rank=None, diff --git a/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py b/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py index 0c724f45aa2..3c641089291 100644 --- a/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py +++ b/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py @@ -19,12 +19,11 @@ from typing import List, Dict, Optional from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker, logger from profiler.advisor.analyzer.schedule.fusion_ops.timeline_api_stack_checker import OpStackFinder -from profiler.advisor.common import constant from profiler.advisor.dataset.dataset import Dataset from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset from profiler.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset -from profiler.cluster_analyse.common_func.file_manager import FileManager - +from profiler.prof_common.file_manager import FileManager +from profiler.prof_common.constant import Constant class AicpuChecker(OperatorChecker): _CHECKER = "aicpu operator" @@ -97,10 +96,10 @@ class AicpuChecker(OperatorChecker): data: Dict[str, Dataset] = {} event_dataset = ComputationAnalysisDataset(collection_path=profiling_data.collection_path, data=data, - task_type=constant.AI_CPU) + task_type=Constant.AI_CPU) # disable multiprocessing, avoid cost time of enable new process for light task - api_stack_finder.get_api_stack_by_op(event_dataset, op_name_list, constant.AI_CPU, + api_stack_finder.get_api_stack_by_op(event_dataset, op_name_list, Constant.AI_CPU, disable_multiprocess=True) return api_stack_finder.get_stack_record() @@ -163,7 +162,7 @@ class AicpuChecker(OperatorChecker): template_dir="templates", template_name="operator_ai_cpu.html", format_result=self.format_operator_result(record, - constant.OPERATOR_LIST_UNLIMIT), + Constant.OPERATOR_LIST_UNLIMIT), add_render_list=add_render_list, priority_background_color=priority, rank=kwargs.get("rank")) @@ -197,7 +196,7 @@ class AicpuChecker(OperatorChecker): return format_result def group_by_list(self, op_list, op_key_list: List = None, - limit: int = constant.OPERATOR_LIST_UNLIMIT): + limit: int = Constant.OPERATOR_LIST_UNLIMIT): if op_list is None: op_list = [] if op_key_list is None: @@ -220,7 +219,7 @@ class AicpuChecker(OperatorChecker): return True def _check_operator(self, op_info) -> bool: - return op_info.task_type == constant.AI_CPU + return op_info.task_type == Constant.AI_CPU class BaserChecker: diff --git a/profiler/advisor/analyzer/computation/bound/block_dim_checker.py b/profiler/advisor/analyzer/computation/bound/block_dim_checker.py index 6eef6f81310..18b46b5176d 100644 --- a/profiler/advisor/analyzer/computation/bound/block_dim_checker.py +++ b/profiler/advisor/analyzer/computation/bound/block_dim_checker.py @@ -16,7 +16,7 @@ import logging from typing import List from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker -from profiler.advisor.common import constant +from profiler.prof_common.constant import Constant from profiler.advisor.config.config import Config from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset @@ -44,7 +44,7 @@ class BlockDimChecker(OperatorChecker): template_dir="templates", template_name="operator_block_dim.html", format_result=self.format_operator_result(record, - constant.OPERATOR_OUT_TOPK), + Constant.OPERATOR_OUT_TOPK), add_render_list=add_render_list, priority_background_color=priority, rank=kwargs.get("rank")) diff --git a/profiler/advisor/analyzer/computation/bound/operator_bound_checker.py b/profiler/advisor/analyzer/computation/bound/operator_bound_checker.py index 9ef64e54694..5e2ee225162 100644 --- a/profiler/advisor/analyzer/computation/bound/operator_bound_checker.py +++ b/profiler/advisor/analyzer/computation/bound/operator_bound_checker.py @@ -16,7 +16,7 @@ import logging from typing import List from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker -from profiler.advisor.common import constant +from profiler.prof_common.constant import Constant from profiler.advisor.config.config import Config from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset from profiler.advisor.utils.utils import to_percent @@ -47,7 +47,7 @@ class OperatorBoundChecker(OperatorChecker): template_dir="templates", template_name="operator_no_bound.html", format_result=self.format_operator_result(record, - constant.OPERATOR_OUT_TOPK), + Constant.OPERATOR_OUT_TOPK), add_render_list=add_render_list, priority_background_color=priority, rank=kwargs.get("rank")) diff --git a/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py b/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py index 6ce417729ad..4ca563e7fd9 100644 --- a/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py +++ b/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py @@ -17,7 +17,6 @@ import logging from typing import List from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker -from profiler.advisor.common import constant from profiler.advisor.config.config import Config from profiler.advisor.dataset.profiling.info_collection import OpInfo from profiler.advisor.result.item import OptimizeItem, StatisticsItem, OptimizeRecord diff --git a/profiler/advisor/analyzer/computation/operator_checker.py b/profiler/advisor/analyzer/computation/operator_checker.py index 17be15b4eb5..73ce3dfffb5 100644 --- a/profiler/advisor/analyzer/computation/operator_checker.py +++ b/profiler/advisor/analyzer/computation/operator_checker.py @@ -17,7 +17,7 @@ import logging from textwrap import fill from typing import List -from profiler.advisor.common import constant +from profiler.prof_common.constant import Constant from profiler.advisor.common.enum_params_parser import EnumParamsParser from profiler.advisor.common.version_control import VersionControl from profiler.advisor.config.config import Config @@ -30,8 +30,8 @@ logger = logging.getLogger() class OperatorChecker(VersionControl): - _SUPPORT_VERSIONS = EnumParamsParser().get_options(constant.CANN_VERSION) - _MAX_TUNE_OP_NUM = constant.OPERATOR_OUT_TOPK + _SUPPORT_VERSIONS = EnumParamsParser().get_options(Constant.CANN_VERSION) + _MAX_TUNE_OP_NUM = Constant.OPERATOR_OUT_TOPK _MIN_TASK_DURATION = 0 _MIN_TASK_DURATION_RATIO = 1.0 _MIN_TOTAL_DURATION_RATIO = 1.0 @@ -105,7 +105,7 @@ class OperatorChecker(VersionControl): self._op_list.sort(key=lambda x: float(x.get_attr("task_duration")), reverse=True) self._tune_op_info_list.sort(key=lambda x: float(x.get_attr("task_duration")), reverse=True) for op in self._op_list: - if op.op_name not in self._tune_op_list and len(self._tune_op_list) < constant.OPERATOR_OUT_TOPK: + if op.op_name not in self._tune_op_list and len(self._tune_op_list) < Constant.OPERATOR_OUT_TOPK: self._tune_op_list.append(op.op_name) return True return False @@ -157,7 +157,7 @@ class OperatorChecker(VersionControl): def is_dynamic_shape(self, profiling_database: ProfilingDataset) -> bool: cann800_major_version = 8 less_than_cann800_list = EnumParamsParser().get_options( - constant.CANN_VERSION, + Constant.CANN_VERSION, filter_func=lambda x: convert_to_float(x.split(".")[0]) < cann800_major_version ) # CANN 8.0.RC1 之前从 ge_info 中获取 op_state 属性，进行动态 shape 逻辑判断 @@ -218,7 +218,7 @@ class OperatorChecker(VersionControl): return format_result def group_by(self, op_list, op_key="op_type", - limit: int = constant.OPERATOR_LIST_UNLIMIT): + limit: int = Constant.OPERATOR_LIST_UNLIMIT): """ group by Profiling.OpInfo's attribute key， then return top limit tuple by duration :param op_list: input a OpInfo list @@ -236,7 +236,7 @@ class OperatorChecker(VersionControl): if summary.get("total_duration"): summary["total_duration"] = float( summary["total_duration"]) + float( - op_info.get_attr("task_duration", constant.DEFAULT_DURATION_ZERO)) + op_info.get_attr("task_duration", Constant.DEFAULT_DURATION_ZERO)) if summary.get("counts"): summary["counts"] += 1 stack_info = op_info.get_attr("stack_info") @@ -248,9 +248,9 @@ class OperatorChecker(VersionControl): else: statistic[op_info.get_attr(op_key)] = {"summary": {}, "op_info_list": []} statistic[op_info.get_attr(op_key)]["summary"]["op_type"] = op_info.get_attr( - "op_type", constant.DEFAULT_OPERATOR_TYPE) + "op_type", Constant.DEFAULT_OPERATOR_TYPE) statistic[op_info.get_attr(op_key)]["summary"]["total_duration"] = float( - op_info.get_attr("task_duration", constant.DEFAULT_DURATION_ZERO)) + op_info.get_attr("task_duration", Constant.DEFAULT_DURATION_ZERO)) statistic[op_info.get_attr(op_key)]["summary"]["counts"] = 1 stack_info = op_info.get_attr("stack_info") if stack_info: diff --git a/profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py b/profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py index 04971cab678..068a209f20b 100644 --- a/profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py +++ b/profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py @@ -24,7 +24,6 @@ from profiler.advisor.interface.interface import Interface from profiler.advisor.utils.utils import ParallelJob, get_analyze_processes from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.common import constant as const logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/dataloader/dataloader_checker.py b/profiler/advisor/analyzer/dataloader/dataloader_checker.py index d4ba7713c70..de9f7d7ca17 100644 --- a/profiler/advisor/analyzer/dataloader/dataloader_checker.py +++ b/profiler/advisor/analyzer/dataloader/dataloader_checker.py @@ -20,7 +20,7 @@ import yaml from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/memory/memory_checker.py b/profiler/advisor/analyzer/memory/memory_checker.py index b446067ef7e..682a19f696d 100644 --- a/profiler/advisor/analyzer/memory/memory_checker.py +++ b/profiler/advisor/analyzer/memory/memory_checker.py @@ -20,7 +20,7 @@ import yaml from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset, MemCollector from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.cluster_analyse.common_func.file_manager import FileManager + logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/overall/environment_variable_checker.py b/profiler/advisor/analyzer/overall/environment_variable_checker.py index 25058a790cc..700946e9040 100644 --- a/profiler/advisor/analyzer/overall/environment_variable_checker.py +++ b/profiler/advisor/analyzer/overall/environment_variable_checker.py @@ -14,7 +14,7 @@ # limitations under the License. import os -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem from profiler.advisor.result.item import OptimizeRecord diff --git a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py b/profiler/advisor/analyzer/overall/overall_summary_analyzer.py index 33dd081bc32..3df71fe8149 100644 --- a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py +++ b/profiler/advisor/analyzer/overall/overall_summary_analyzer.py @@ -19,9 +19,8 @@ from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.display.html.render import HTMLRender from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.result.result import OptimizeResult -from profiler.compare_tools.compare_backend.utils.constant import Constant from profiler.compare_tools.compare_interface.comparison_interface import ComparisonInterface - +from profiler.prof_common.constant import Constant class OverallSummaryAnalyzer(BaseAnalyzer): OVERALL_SUMMARY_ANALYZER = "overall summary" diff --git a/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py b/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py index 126fe30176c..ed0500d3b23 100644 --- a/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py +++ b/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py @@ -16,7 +16,7 @@ # limitations under the License. import logging -from profiler.advisor.common import constant as const +from profiler.prof_common.constant import Constant from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.item import OptimizeItem, OptimizeRecord @@ -60,11 +60,11 @@ class OpDispatchAnalyzer(BaseAnalyzer): """ if hasattr(event_dataset, "ops_compile"): self._op_compile = getattr(event_dataset, "ops_compile") - if not self._op_compile or self._op_compile.total_count < const.MAX_OP_COMPILE_NUM: + if not self._op_compile or self._op_compile.total_count < Constant.MAX_OP_COMPILE_NUM: return self._issues_record.append(['operator dispatch', - const.OP_COMPILE_ID, + Constant.OP_COMPILE_ID, self._op_compile.total_count, self._op_compile.total_time]) else: diff --git a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py b/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py index 7407823106e..80ac8006155 100644 --- a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py +++ b/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py @@ -20,7 +20,7 @@ import re from tqdm import tqdm from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.common import constant as const +from profiler.prof_common.constant import Constant from profiler.advisor.common.analyzer_scopes import SupportedScopes from profiler.advisor.common.timeline.event import TimelineEvent from profiler.advisor.config.config import Config @@ -48,13 +48,13 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): return PriorityBackgroundColor.low def optimize(self, **kwargs): - disable_affinity_api = os.getenv(const.DISABLE_AFFINITY_API) + disable_affinity_api = os.getenv(Constant.DISABLE_AFFINITY_API) if disable_affinity_api is not None and disable_affinity_api.lower() == "true": logger.info( "Skip affinity api analysis due to longer processing time due to env 'DISABLE_AFFINITY_API'") return self.result - for mode in [const.ATEN.lower(), const.OPTIMIZER.lower()]: + for mode in [Constant.ATEN.lower(), Constant.OPTIMIZER.lower()]: for op_combined, npu_apis in tqdm(getattr(init_timeline_ops_db(self.cann_version, self.torch_version), f"_{mode}_op_api_map").items(), leave=False, ncols=100, @@ -98,7 +98,7 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): suggestion = "Please replace training api according to sub table 'Affinity training api'" if self.empty_stacks: desc += ", but with no stack" - suggestion = const.TIMELINE_EMPTY_STACKS_PROMPT.format( + suggestion = Constant.TIMELINE_EMPTY_STACKS_PROMPT.format( timeline_profiling_doc_url=Config().timeline_with_stack_doc_url ) @@ -148,7 +148,7 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): for op_rule, stack in op_stack.items(): if op_rule not in self.matched_op_stacks: self.matched_op_stacks[op_rule] = {} - if stack == const.TIMELINE_FUSION_OPS_NO_STACK_FLAG: + if stack == Constant.TIMELINE_FUSION_OPS_NO_STACK_FLAG: continue if stack not in self.matched_op_stacks[op_rule]: self.matched_op_stacks[op_rule][stack] = 0 @@ -162,7 +162,7 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): :Param npu_api: api of torch_npu, generally more efficient than torch api :Param mode: aten or dequeue or optimizer """ - op_list = ops.split(const.OP_SEP) + op_list = ops.split(Constant.OP_SEP) matched_op_index = set() api_ops_matched = False @@ -190,7 +190,7 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): :Param mode: aten or dequeue or optimizer """ matched_op_index = set() - total_op_name = "".join([f"{const.OP_SEP}{self._replace_op_name_prefix(event.name, mode)}{const.OP_SEP}" + total_op_name = "".join([f"{Constant.OP_SEP}{self._replace_op_name_prefix(event.name, mode)}{Constant.OP_SEP}" for event in getattr(event_dataset, mode)]) matched_pattern_index_tuple = [(x.start(0), x.end(0)) for x in re.finditer(op_rule_pattern, total_op_name)] @@ -211,7 +211,7 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): # by the regex index and then calculate the real index for matched fusion operators in event dataset for left, right in zip(total_ops_split_points, total_ops_split_points[1:]): matched_op_flag = True if (left, right) in matched_pattern_index_tuple else False - matched_ops_list = total_op_name[left: right].strip(const.OP_SEP).split(const.OP_SEP + const.OP_SEP) + matched_ops_list = total_op_name[left: right].strip(Constant.OP_SEP).split(Constant.OP_SEP + Constant.OP_SEP) op_index.append([matched_op_flag, len(matched_ops_list)]) for i, _ in enumerate(op_index): if i > 0: @@ -243,10 +243,10 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): continue matched_op_rules.append(op_rule) - stack = event.args.get(const.CALL_STACKS) + stack = event.args.get(Constant.CALL_STACKS) if not stack: - logger.debug("Got empty '%s' for event %s", const.CALL_STACKS, event) + logger.debug("Got empty '%s' for event %s", Constant.CALL_STACKS, event) continue if self.empty_stacks and stack: @@ -256,17 +256,17 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): if matched_op_rules and not stack_record: for op_rule in matched_op_rules: - stack_record[op_rule] = const.TIMELINE_FUSION_OPS_NO_STACK_FLAG + stack_record[op_rule] = Constant.TIMELINE_FUSION_OPS_NO_STACK_FLAG return stack_record def _replace_op_name_prefix(self, event_name, mode): - if mode == const.DEQUEUE.lower(): - op_name_prefix = f"{const.DEQUEUE}{const.DEQUEUE_SEP}" - elif mode == const.ATEN: - op_name_prefix = f"{const.ATEN}{const.ATEN_SEP}" + if mode == Constant.DEQUEUE.lower(): + op_name_prefix = f"{Constant.DEQUEUE}{Constant.DEQUEUE_SEP}" + elif mode == Constant.ATEN: + op_name_prefix = f"{Constant.ATEN}{Constant.ATEN_SEP}" else: - op_name_prefix = f"{const.OPTIMIZER}.{const.OPTIMIZER_STEP}{const.OPTIMIZER_SEP}" + op_name_prefix = f"{Constant.OPTIMIZER}.{Constant.OPTIMIZER_STEP}{Constant.OPTIMIZER_SEP}" return event_name.replace(op_name_prefix, "") @@ -283,7 +283,7 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): return op_rule, enable_regex enable_regex = True - op_pattern_list = op_rule.split(const.OP_SEP) + op_pattern_list = op_rule.split(Constant.OP_SEP) format_op_pattern = "" for op_pattern in op_pattern_list: matched_res = re.search(r'\((\w*?)\)', op_pattern) @@ -294,7 +294,7 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): op_names = op_pattern[ops_index_range[0]: ops_index_range[1]] tmp_op_names_record = [] for op_name in op_names.split("|"): - tmp_op_names_record.append(f"{const.OP_SEP}{op_name.strip(' ')}{const.OP_SEP}") + tmp_op_names_record.append(f"{Constant.OP_SEP}{op_name.strip(' ')}{Constant.OP_SEP}") op_suffix = op_pattern[ops_index_range[1] + 1:] op_names_format = f"({'|'.join(tmp_op_names_record)}){op_suffix}" diff --git a/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py b/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py index 126584c3d73..72c302e1efc 100644 --- a/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py +++ b/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py @@ -15,7 +15,7 @@ import logging from typing import List -from profiler.advisor.common import constant as const +from profiler.prof_common.constant import Constant from profiler.advisor.common.timeline.event import TimelineEvent from profiler.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset from profiler.advisor.result.result import OptimizeResult @@ -41,16 +41,16 @@ class OpStackFinder: dst_op_event = event_dataset.ops_with_stack.get(dst_op_event_key) if not dst_op_event: - return const.TIMELINE_BACKWARD_NO_STACK_CODE + return Constant.TIMELINE_BACKWARD_NO_STACK_CODE return int(dst_op_event.get("dataset_index")) @staticmethod def _query_index_by_acl_to_npu(acl_to_npu_event): if acl_to_npu_event: - return const.TIMELINE_ACL_TO_NPU_NO_STACK_CODE + return Constant.TIMELINE_ACL_TO_NPU_NO_STACK_CODE - return const.TIMELINE_BACKWARD_NO_STACK_CODE + return Constant.TIMELINE_BACKWARD_NO_STACK_CODE def get_api_stack_by_op(self, event_dataset: ComputationAnalysisDataset, op_name: List[str] = None, task_type: str = None, @@ -126,7 +126,7 @@ class OpStackFinder: def _get_api_stack_by_op(self, event_dataset: ComputationAnalysisDataset, op_name: str, task_type: str): for _, src_op_event in event_dataset.ops_with_task_type.items(): - op_task_type = src_op_event.get(const.TASK_TYPE) + op_task_type = src_op_event.get(Constant.TASK_TYPE) if not (src_op_event.name == op_name and op_task_type and op_task_type == task_type): continue @@ -163,8 +163,8 @@ class OpStackFinder: if task_type is not None: self._get_api_stack_by_op(event_dataset, op_name, task_type) else: - self._get_api_stack_by_op(event_dataset, op_name, const.AI_CORE) - self._get_api_stack_by_op(event_dataset, op_name, const.AI_CPU) + self._get_api_stack_by_op(event_dataset, op_name, Constant.AI_CORE) + self._get_api_stack_by_op(event_dataset, op_name, Constant.AI_CPU) def _format_stack_record(self): stack_list = [] @@ -176,13 +176,13 @@ class OpStackFinder: if index not in self.matched_index: return None event = TimelineEvent(event) - stack = event.args.get(const.CALL_STACKS) + stack = event.args.get(Constant.CALL_STACKS) - stack = stack if stack else const.NO_STACK_REASON_MAP.get(const.TIMELINE_BACKWARD_NO_STACK_CODE) + stack = stack if stack else Constant.NO_STACK_REASON_MAP.get(Constant.TIMELINE_BACKWARD_NO_STACK_CODE) for matched_op_info in self._task_id_record.get(index, []): self._stack_record.append([*matched_op_info, stack]) - for matched_op_info in self._task_id_record.get(const.TIMELINE_ACL_TO_NPU_NO_STACK_CODE, []): + for matched_op_info in self._task_id_record.get(Constant.TIMELINE_ACL_TO_NPU_NO_STACK_CODE, []): self._stack_record.append([*matched_op_info, - const.NO_STACK_REASON_MAP.get(const.TIMELINE_ACL_TO_NPU_NO_STACK_CODE)]) + Constant.NO_STACK_REASON_MAP.get(Constant.TIMELINE_ACL_TO_NPU_NO_STACK_CODE)]) return None diff --git a/profiler/advisor/analyzer/schedule/gc/gc_checker.py b/profiler/advisor/analyzer/schedule/gc/gc_checker.py index be1a6053677..321b80fc6dc 100644 --- a/profiler/advisor/analyzer/schedule/gc/gc_checker.py +++ b/profiler/advisor/analyzer/schedule/gc/gc_checker.py @@ -20,8 +20,8 @@ from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisData from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.utils.utils import convert_to_float, convert_to_int, safe_division -from profiler.advisor.common import constant as const -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.constant import Constant +from profiler.prof_common.file_manager import FileManager logger = logging.getLogger() @@ -120,7 +120,7 @@ class GcChecker: while free_event_index < len(large_free_events) and acl_event_index < len(acl_events): free_event = large_free_events[free_event_index] - free_event_name = f"{const.FREE}-{free_event_index}" + free_event_name = f"{Constant.FREE}-{free_event_index}" free_event_start_time = convert_to_float(free_event.ts) free_event_end_time = free_event_start_time + convert_to_float(free_event.dur) if free_event_name not in free_include_acl_events: @@ -153,7 +153,7 @@ class GcChecker: event_indexs = range(len(large_free_events)) for index, free_event in sorted(zip(event_indexs, large_free_events), key=lambda x: x[1].dur, reverse=True): - free_event_name = f"{const.FREE}-{index}" + free_event_name = f"{Constant.FREE}-{index}" free_duration = convert_to_float(free_event.dur) acl_event_dur = free_include_acl_events.get(free_event_name, {}).get("acl_event_dur", 0.0) acl_event_count = free_include_acl_events.get(free_event_name, {}).get("acl_event_count", 0) diff --git a/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py b/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py index e4cf2430c50..878d5869ed6 100644 --- a/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py +++ b/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py @@ -18,7 +18,7 @@ import os from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py index c27b398f460..62dc005ece8 100644 --- a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py +++ b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py @@ -16,14 +16,14 @@ import logging import os from profiler.advisor.analyzer.schedule.timeline_base_checker import TimelineBaseChecker -from profiler.advisor.common import constant as const +from profiler.prof_common.constant import Constant from profiler.advisor.config.config import Config from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.utils.utils import format_timeline_result, safe_division -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager logger = logging.getLogger() @@ -51,13 +51,13 @@ class SynchronizeStreamChecker(TimelineBaseChecker): synchronize_num = 0 synchronize_stream = event_dataset.synchronize_stream for index, op in enumerate(synchronize_stream): - if op.name.startswith(const.NODE_LAUNCH): + if op.name.startswith(Constant.NODE_LAUNCH): node_launch_num += 1 - if op.name.startswith(const.SYNC_STREAM): + if op.name.startswith(Constant.SYNC_STREAM): synchronize_num += 1 # 统计nodeLaunch 和 synchronizeStream 一前一后连续出现次数 - if index > 0 and synchronize_stream[index - 1].name.startswith(const.NODE_LAUNCH): + if index > 0 and synchronize_stream[index - 1].name.startswith(Constant.NODE_LAUNCH): co_occurrence_num += 1 # 当共现次数很多时，则大概率设置了ASCEND_LAUNCH_BLOCKING环境变量 diff --git a/profiler/advisor/analyzer/schedule/timeline_base_checker.py b/profiler/advisor/analyzer/schedule/timeline_base_checker.py index 9ef492c1a24..7d3cdf7e90b 100644 --- a/profiler/advisor/analyzer/schedule/timeline_base_checker.py +++ b/profiler/advisor/analyzer/schedule/timeline_base_checker.py @@ -16,7 +16,7 @@ from abc import ABC, abstractmethod import multiprocessing import logging -from profiler.advisor.common import constant as const +from profiler.prof_common.constant import Constant from profiler.advisor.common.timeline.event import TimelineEvent from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.result import OptimizeResult @@ -46,7 +46,7 @@ class TimelineBaseChecker(ABC): for op, stack in op_stack.items(): if op not in self.matched_op_stacks: self.matched_op_stacks[op] = {} - if stack == const.TIMELINE_FUSION_OPS_NO_STACK_FLAG: + if stack == Constant.TIMELINE_FUSION_OPS_NO_STACK_FLAG: continue if stack not in self.matched_op_stacks[op]: self.matched_op_stacks[op][stack] = 0 @@ -62,15 +62,15 @@ class TimelineBaseChecker(ABC): continue matched_ops.append(op) - stack = event.args.get(const.CALL_STACKS) + stack = event.args.get(Constant.CALL_STACKS) if not stack: - logger.debug("Got empty '%s' for event %s", const.CALL_STACKS, event) + logger.debug("Got empty '%s' for event %s", Constant.CALL_STACKS, event) continue if not self._is_keep_stack(stack): self.framework_black_list = True - logger.debug("Drop stack from framework %s", const.FRAMEWORK_STACK_BLACK_LIST) + logger.debug("Drop stack from framework %s", Constant.FRAMEWORK_STACK_BLACK_LIST) continue if self.empty_stacks and stack: @@ -80,7 +80,7 @@ class TimelineBaseChecker(ABC): if matched_ops and not stack_record: for op in matched_ops: - stack_record[op] = const.TIMELINE_FUSION_OPS_NO_STACK_FLAG + stack_record[op] = Constant.TIMELINE_FUSION_OPS_NO_STACK_FLAG return stack_record @@ -91,7 +91,7 @@ class TimelineBaseChecker(ABC): return False final_called_stack = stack_list[0] - for framework in const.FRAMEWORK_STACK_BLACK_LIST: + for framework in Constant.FRAMEWORK_STACK_BLACK_LIST: if framework in final_called_stack.split("/"): return False return True diff --git a/profiler/advisor/common/constant.py b/profiler/advisor/common/constant.py deleted file mode 100644 index dbdfbde0b19..00000000000 --- a/profiler/advisor/common/constant.py +++ /dev/null @@ -1,161 +0,0 @@ -# Copyright (c) 2023, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import stat - -# timeline -DEQUEUE = "Dequeue" -DEQUEUE_SEP = "@" -ATEN = "aten" -NPU = "npu" -ATEN_SEP = "::" -OPTIMIZER = "Optimizer" -OPTIMIZER_SEP = "#" -OPTIMIZER_STEP = "step" -ENQUEUE = "enqueue" -TORCH_TO_NPU = "torch_to_npu" -FREE = "free" -OP_COMPILE_NAME = "AscendCL@aclopCompileAndExecute" -OP_COMPILE_ID = "aclopCompileAndExecute" -SYNC_STREAM = "AscendCL@aclrtSynchronizeStream" -NODE_LAUNCH = "Node@launch" -MAX_OP_COMPILE_NUM = 20 -ACL_TO_NPU = "acl_to_npu" -TASK_TYPE = "Task Type" -CPU_OP = "cpu_op" -AI_CORE = "AI_CORE" -AI_CPU = "AI_CPU" -MIX_AIC = "MIX_AIC" -CALL_STACKS = "Call stack" -INPUT_DIMS = "Input Dims" -OP_SEP = "-" -ADVISOR_MAX_PROCESSES = 8 -ADVISOR_ANALYZE_PROCESSES = "ADVISOR_ANALYZE_PROCESSES" -TIMELINE_OP_STACKS_DATASET = "timeline_op_stacks_dataset" -TIMELINE_BACKWARD_NO_STACK = "Backward broadcast, without call stacks in profiling." -TIMELINE_ACL_TO_NPU_NO_STACK = "Incoming flow is 'acl_to_npu', without call stacks in profiling." -TIMELINE_BACKWARD_NO_STACK_CODE = -1 -TIMELINE_ACL_TO_NPU_NO_STACK_CODE = -2 -TIMELINE_FUSION_OPS_NO_STACK_FLAG = "NO STACK" -NO_STACK_REASON_MAP = { - TIMELINE_BACKWARD_NO_STACK_CODE: "Backward broadcast, without call stacks in profiling.", - TIMELINE_ACL_TO_NPU_NO_STACK_CODE: "Incoming flow is 'acl_to_npu', without call stacks in profiling." -} -AFFINITY_TRAINING_API = "Affinity training api" -TIMELINE_EMPTY_STACKS_PROMPT = "These APIs have no code stack. If parameter 'with_stack=False' while profiling, " \ - "please refer to {timeline_profiling_doc_url} to set 'with_stack=True'. " \ - "Otherwise, ignore following affinity APIs due to backward broadcast lack of stack." - -CLUSTER_ANALYSIS = "Cluster analysis" -SLOW_RANK_TIME_RATIO_THRESHOLD = 0.05 - -CANN_VERSION = "cann_version" -TORCH_VERSION = "torch_version" -PROFILING_TYPE = "profiling_type" -ANALYSIS_DIMENSIONS = "analysis_dimensions" - -PROFILER_METADATA = "profiler_metadata.json" - -TERMINAL_OUTPUT_HEADERS = ["No.", "Problem", "Description", "Suggestion"] -SKIP_ANALYZE_PROMPT = "Finish analysis, no optimization suggestions" -SKIP_QUERY_PROMPT = "Finish query operator stack, no operators" - -# operator output constant -OPERATOR_OUT_TOPK = 10 -OPERATOR_LIST_UNLIMIT = -1 - -DEFAULT_OPERATOR_TYPE = 'None_type' -DEFAULT_DURATION_ZERO = 0.0 - -ADVISOR_LOG_LEVEL = "ADVISOR_LOG_LEVEL" -DEFAULT_LOG_LEVEL = "INFO" -SUPPORTED_LOG_LEVEL = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] - -RULE_BUCKET = "RULE-BUCKET" -CLOUD_RULE_REGION_CN_NORTH_9 = "cn-north-9" -CLOUD_RULE_REGION_CN_NORTH_7 = "cn-north-7" -CLOUD_RULE_REGION_CN_SOUTHWEST_2 = "cn-southwest-2" -CLOUD_RULE_REGION_LIST = [CLOUD_RULE_REGION_CN_NORTH_7, CLOUD_RULE_REGION_CN_NORTH_9, CLOUD_RULE_REGION_CN_SOUTHWEST_2] -INNER_REGION_LIST = [CLOUD_RULE_REGION_CN_NORTH_7] -DEFAULT_CLOUD_RULE_REGION = CLOUD_RULE_REGION_CN_SOUTHWEST_2 - -HTTP_PREFIXES = "http://" -HTTPS_PREFIXES = "https://" -COMMON_YAML_DIR = "modelarts/solution/ma_advisor_rules/" -COMMON_ENDPOINT_SUFFIX = "obs.{}.myhuaweicloud.com" -INNER_ENDPOINT_SUFFIX = "obs.{}.ulanqab.huawei.com" - -AICPU_RULES_YAML_NAME = "aicpu_rules.yaml" -FUSION_PASS_YAML_NAME = "op_fusion_pass.yaml" -TIMELINE_FUSION_OPS_YAML_NAME = "timeline_fusion_ops.yaml" -CLOUD_YAML_NAME_LIST = [AICPU_RULES_YAML_NAME, FUSION_PASS_YAML_NAME, TIMELINE_FUSION_OPS_YAML_NAME] - -MAX_RETRIES = 3 -TIMEOUT = 3 -DEPTH_LIMIT = 20 - -ADVISOR_RULE_PATH = "ADVISOR_RULE_PATH" -CLOUD_RULE_PATH = "rules/cloud/" -DEFAULT_RULE_PATH = "./rules/" - -TIMELINE_FUSION_OPS_INVALID_UNIQUE_ID = -1 - -DEFAULT_TEMPLATE_HEADER = "Performance Optimization Suggestions" - -PT_PROF_SUFFIX = "ascend_pt" -ASCEND_PROFILER_OUTPUT = "ASCEND_PROFILER_OUTPUT" -COLLECTION_PATH = "collection_path" -CLUSTER_ANALYSIS_OUTPUT = "cluster_analysis_output" -KERNEL_DETAILS_CSV = "kernel_details.csv" -CLUSTER_STEP_TIME_CSV = "cluster_step_trace_time.csv" -CLUSTER_COMM_JSON = "cluster_communication.json" -COMMUNICATION_JSON = "communication.json" - -BOTTLENECK = "bottleneck" -DATA = "data" -ADVISOR_ANALYSIS_OUTPUT_DIR = "advisor_analysis_result" -DEFAULT_PROCESSES = 8 -CLUSTER_ANALYSIS_FILE_PATTERN = [ - r'profiler_info_\d+\.json', "step_trace_time.csv", "communication.json", "communication_matrix.json" -] -ANALYSIS_OUTPUT_PATH = "ANALYSIS_OUTPUT_PATH" -DEFAULT_RANK_FOR_PROFILING_ANALYSIS = 0 -PROFILER_INFO_FILE_PATTERN = r"profiler_info_(\d+)\.json" -DISABLE_STREAMINIG_READER = "DISABLE_STREAMINIG_READER" -FRAMEWORK_STACK_BLACK_LIST = ["torch", "torch_npu", "megatron", "deepspeed"] -DISABLE_STREAMING_READER = "DISABLE_STREAMING_READER" -MAX_FILE_SIZE = 10 ** 10 -MAX_NUM_PROCESSES = 4 -DEFAULT_STEP = "-1" -STEP_RANK_SEP = "_" - -MAX_READ_LINE_BYTES = 8196 * 1024 -MAX_READ_FILE_BYTES = 64 * 1024 * 1024 * 1024 -MAX_READ_DB_FILE_BYTES = 8 * 1024 * 1024 * 1024 - -# Unit Conversion -COMMUNICATION_B_TO_GB = 0.001 ** 3 -US_TO_S = 0.001 ** 2 - -WRITE_MODES = stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP -WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC - -DISABLE_PROFILING_COMPARISON = "DISABLE_PROFILING_COMPARISON" -FREE_DURATION_FOR_GC_ANALYSIS = "FREE_DURATION_FOR_GC_ANALYSIS" -DISABLE_AFFINITY_API = "DISABLE_AFFINITY_API" - -# communication.json -TOTAL_OP_INFO = "Total Op Info" diff --git a/profiler/advisor/common/enum_params_parser.py b/profiler/advisor/common/enum_params_parser.py index ea41620a9c0..e9a3f6639cd 100644 --- a/profiler/advisor/common/enum_params_parser.py +++ b/profiler/advisor/common/enum_params_parser.py @@ -20,7 +20,7 @@ import typing from profiler.advisor.common.timeline.event import AdvisorDict from profiler.advisor.utils.utils import singleton -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/advisor/common/graph/graph_parser.py b/profiler/advisor/common/graph/graph_parser.py index a89cf738fff..cffbfa19b4e 100644 --- a/profiler/advisor/common/graph/graph_parser.py +++ b/profiler/advisor/common/graph/graph_parser.py @@ -21,7 +21,7 @@ from collections import deque from dataclasses import dataclass from typing import List, Tuple, Dict -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager from profiler.advisor.utils.file import FileOpen logger = logging.getLogger() diff --git a/profiler/advisor/common/timeline/fusion_ops_db.py b/profiler/advisor/common/timeline/fusion_ops_db.py index ad8b5981c72..171786b15c5 100644 --- a/profiler/advisor/common/timeline/fusion_ops_db.py +++ b/profiler/advisor/common/timeline/fusion_ops_db.py @@ -17,13 +17,13 @@ import logging import os -from profiler.advisor.common import constant +from profiler.prof_common.constant import Constant from profiler.advisor.common.enum_params_parser import EnumParamsParser from profiler.advisor.common.timeline.fusion_ops_rule import OpRule from profiler.advisor.common.timeline.fusion_ops_rule_handler import TimelineOpRuleHandler from profiler.advisor.utils.log import get_log_level from profiler.advisor.utils.utils import get_file_path_by_walk -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager logger = logging.getLogger() logger.setLevel(get_log_level()) @@ -37,25 +37,25 @@ def init_timeline_ops_db(cann_version=None, torch_version=None): def get_timeline_fusion_ops_yaml_path(): # 环境变量 ADVISOR_RULE_PATH 不为空且该路径存在, os.walk遍历其下文件, 若存在相应的规则文件则返回路径 - advisor_rule_path = os.getenv(constant.ADVISOR_RULE_PATH) + advisor_rule_path = os.getenv(Constant.ADVISOR_RULE_PATH) if advisor_rule_path and os.path.exists(advisor_rule_path): - specified_file_path = get_file_path_by_walk(advisor_rule_path, constant.TIMELINE_FUSION_OPS_YAML_NAME) + specified_file_path = get_file_path_by_walk(advisor_rule_path, Constant.TIMELINE_FUSION_OPS_YAML_NAME) if len(specified_file_path.strip()) and os.path.exists(specified_file_path): logger.debug("Successfully find The %s file which is specified by the environment variable: %s.", - specified_file_path, constant.ADVISOR_RULE_PATH) + specified_file_path, Constant.ADVISOR_RULE_PATH) return specified_file_path logger.warning("The %s does not exist in path: %s. Try to use cloud or default local YAML file.", - constant.TIMELINE_FUSION_OPS_YAML_NAME, os.path.normpath(advisor_rule_path)) + Constant.TIMELINE_FUSION_OPS_YAML_NAME, os.path.normpath(advisor_rule_path)) # 检查云文件默认保存路径文件夹下是否存在相应文件, 默认路径 ~/rules/cloud/ - cloud_file_path = os.path.join(os.path.expanduser("~"), constant.CLOUD_RULE_PATH, - constant.TIMELINE_FUSION_OPS_YAML_NAME) + cloud_file_path = os.path.join(os.path.expanduser("~"), Constant.CLOUD_RULE_PATH, + Constant.TIMELINE_FUSION_OPS_YAML_NAME) if os.path.exists(cloud_file_path): - logger.debug("Successfully find The cloud %s file in %s.", constant.TIMELINE_FUSION_OPS_YAML_NAME, + logger.debug("Successfully find The cloud %s file in %s.", Constant.TIMELINE_FUSION_OPS_YAML_NAME, cloud_file_path) return cloud_file_path # 检查本地默认文件 local_file_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), - constant.DEFAULT_RULE_PATH, constant.TIMELINE_FUSION_OPS_YAML_NAME) + Constant.DEFAULT_RULE_PATH, Constant.TIMELINE_FUSION_OPS_YAML_NAME) if not os.path.exists(local_file_path): # 若本地默认文件不存在, 则log异常信息并 logger.error("The default local YAML file does not exist. Please check the YAML file in the default path %s.", @@ -68,8 +68,8 @@ class FusionOperatorDB: def __init__(self, file_path=None, cann_version=None, torch_version=None): self.timeline_fusion_ops_yaml_path = os.path.normpath(get_timeline_fusion_ops_yaml_path()) - self.cann_version = cann_version or EnumParamsParser().get_default(constant.CANN_VERSION) - self.torch_version = torch_version or EnumParamsParser().get_default(constant.TORCH_VERSION) + self.cann_version = cann_version or EnumParamsParser().get_default(Constant.CANN_VERSION) + self.torch_version = torch_version or EnumParamsParser().get_default(Constant.TORCH_VERSION) self._supported_version_dict = {} @@ -110,9 +110,9 @@ class FusionOperatorDB: return self._optimizer_op_api_map def get_fusion_operator_with_unique_id(self, unique_id): - if unique_id == constant.TIMELINE_FUSION_OPS_INVALID_UNIQUE_ID: + if unique_id == Constant.TIMELINE_FUSION_OPS_INVALID_UNIQUE_ID: logger.warning("The specified unique id: %s is invalid.Please check whether the rule of the unique id " - "exists and modify the rule.", constant.TIMELINE_FUSION_OPS_INVALID_UNIQUE_ID) + "exists and modify the rule.", Constant.TIMELINE_FUSION_OPS_INVALID_UNIQUE_ID) return {} result_tmp_rule = self.timeline_op_rule_handler.get_tmp_timeline_op_rule_with_unique_id(unique_id) result_op_rule = OpRule(result_tmp_rule) @@ -195,7 +195,7 @@ class FusionOperatorDB: for key_unique_id, supported_version in self._supported_version_dict.items(): if self._is_version_supported_in_versions(supported_version, cann_version, torch_version): return key_unique_id - return constant.TIMELINE_FUSION_OPS_INVALID_UNIQUE_ID + return Constant.TIMELINE_FUSION_OPS_INVALID_UNIQUE_ID def _is_version_supported_in_versions(self, supported_version, cann_version=None, torch_version=None): """校验当前cann版本和torch版本是否存在在规则库中的版本支持数组的元素中""" @@ -216,9 +216,9 @@ class FusionOperatorDB: def _parse_db(self): """生成输出的规则库""" - self._parse(constant.ATEN) - self._parse(constant.DEQUEUE) - self._parse(constant.OPTIMIZER) + self._parse(Constant.ATEN) + self._parse(Constant.DEQUEUE) + self._parse(Constant.OPTIMIZER) def _parse(self, mode): """生成输出的规则库中指定部分， 如aten, Optimizer等""" @@ -252,7 +252,7 @@ class FusionOperatorDB: if not os.path.exists(file_path): logger.warning("Path: '%s' does not exist, please specific existed path of " "fusion operators yaml file by setting env '%s'", - os.path.abspath(file_path), constant.ADVISOR_RULE_PATH) + os.path.abspath(file_path), Constant.ADVISOR_RULE_PATH) self.is_empty = True return {} diff --git a/profiler/advisor/common/timeline/fusion_ops_rule_handler.py b/profiler/advisor/common/timeline/fusion_ops_rule_handler.py index b0558cca6d9..25bbdf100df 100644 --- a/profiler/advisor/common/timeline/fusion_ops_rule_handler.py +++ b/profiler/advisor/common/timeline/fusion_ops_rule_handler.py @@ -2,7 +2,7 @@ import copy import logging -from profiler.advisor.common import constant +from profiler.prof_common.constant import Constant from profiler.advisor.common.timeline.fusion_ops_rule import OpRule from profiler.advisor.utils.log import get_log_level @@ -189,5 +189,5 @@ class TimelineOpRuleHandler: logger.error("Advise to use a positive integer as the unique id of rules. " "Negative numbers: %s are not recommended to use as unique id. " "If specified invalid unique id: %s is used, an empty rule is returned by default.", - unique_id, constant.TIMELINE_FUSION_OPS_INVALID_UNIQUE_ID) + unique_id, Constant.TIMELINE_FUSION_OPS_INVALID_UNIQUE_ID) return self._all_tmp_timeline_op_rule.get(unique_id) diff --git a/profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py b/profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py index db31c1a0c5f..d7aab88b935 100644 --- a/profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py +++ b/profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py @@ -22,7 +22,6 @@ import traceback import ijson from tqdm import tqdm -from profiler.advisor.common import constant as const from profiler.advisor.common.timeline.event import TimelineEvent from profiler.advisor.utils.utils import get_file_path_from_directory from profiler.advisor.utils.utils import convert_to_float, parse_json_with_generator diff --git a/profiler/advisor/dataset/cluster/cluster_dataset.py b/profiler/advisor/dataset/cluster/cluster_dataset.py index 66bf993a2f1..71cdb9777d9 100644 --- a/profiler/advisor/dataset/cluster/cluster_dataset.py +++ b/profiler/advisor/dataset/cluster/cluster_dataset.py @@ -20,9 +20,8 @@ import re from collections import defaultdict from profiler.advisor.dataset.dataset import Dataset from profiler.advisor.utils.utils import singleton -from profiler.cluster_analyse.common_func.file_manager import FileManager -from profiler.advisor.common import constant as const -from profiler.cluster_analyse.common_func.constant import Constant +from profiler.prof_common.file_manager import FileManager +from profiler.prof_common.constant import Constant from profiler.cluster_analyse.cluster_analysis import Interface from profiler.advisor.dataset.cluster.cluster_step_trace_time_bean import ClusterStepTraceTimeBean from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo @@ -61,7 +60,7 @@ class ClusterDataset(Dataset): raise ValueError(f"Cluster analyze backend failed:{e}") from e def load_csv_data(self, file_name, data_bean): - csv_path = os.path.join(self.collection_path, const.CLUSTER_ANALYSIS_OUTPUT, file_name) + csv_path = os.path.join(self.collection_path, Constant.CLUSTER_ANALYSIS_OUTPUT, file_name) if not os.path.exists(csv_path): msg = "[ERROR] cluster_step_trace_time.csv doesn't exist, terminate analysis." raise RuntimeError(msg) @@ -69,7 +68,7 @@ class ClusterDataset(Dataset): return data def load_json_data(self, file_name): - json_path = os.path.join(self.collection_path, const.CLUSTER_ANALYSIS_OUTPUT, file_name) + json_path = os.path.join(self.collection_path, Constant.CLUSTER_ANALYSIS_OUTPUT, file_name) if not os.path.exists(json_path): msg = "[ERROR] cluster_communication.json doesn't exist, terminate analysis." raise RuntimeError(msg) @@ -92,14 +91,14 @@ class ClusterStepTraceTimeDataset(ClusterDataset): for step_bean in step_data: if step_bean.type == self.RANK: step_rank_record = [] - step = str(step_bean.step).replace(" ", "") or str(const.DEFAULT_STEP) + step = str(step_bean.step).replace(" ", "") or str(Constant.DEFAULT_STEP) rank = str(step_bean.index).replace(" ", "") if step: step_rank_record.append(step) if rank: step_rank_record.append(rank) - step_rank_index = const.STEP_RANK_SEP.join(step_rank_record) + step_rank_index = Constant.STEP_RANK_SEP.join(step_rank_record) step_dict[step_rank_index][0] += step_bean.compute step_dict[step_rank_index][1] += step_bean.communication step_dict[step_rank_index][2] += step_bean.free @@ -119,7 +118,7 @@ class ClusterStepTraceTimeDataset(ClusterDataset): def _parse(self): self.cluster_analyze() try: - step_data = self.load_csv_data(const.CLUSTER_STEP_TIME_CSV, ClusterStepTraceTimeBean) + step_data = self.load_csv_data(Constant.CLUSTER_STEP_TIME_CSV, ClusterStepTraceTimeBean) except RuntimeError as e: logger.error("捕获到异常：%s", e) self._step_dict = None @@ -168,7 +167,7 @@ class ClusterCommunicationDataset(ClusterDataset): self.hccl_dict.setdefault(comm_group, defaultdict(lambda: defaultdict(list))) for step, step_dict in group_dict.items(): for op, op_dict in step_dict.items(): - self.compute_bandwidth(step.lower().lstrip("step") or str(const.DEFAULT_STEP), op_dict) + self.compute_bandwidth(step.lower().lstrip("step") or str(Constant.DEFAULT_STEP), op_dict) self.process_hccl_info(comm_group, step, op, op_dict) def process_hccl_info(self, group, step, op, op_dict): @@ -194,14 +193,14 @@ class ClusterCommunicationDataset(ClusterDataset): raise ValueError(msg) from e for comm_type, bw_dict in rank_dict.get(self.COMMUNICATION_BANDWIDTH_INFO, {}).items(): if comm_type == self.SDMA: - self.rank_bw_dict[f"{step}{const.STEP_RANK_SEP}{rank}"][self.SDMA_SIZE_MB] += \ + self.rank_bw_dict[f"{step}{Constant.STEP_RANK_SEP}{rank}"][self.SDMA_SIZE_MB] += \ bw_dict.get(self.TRANSIT_SIZE) - self.rank_bw_dict[f"{step}{const.STEP_RANK_SEP}{rank}"][self.SDMA_TIME_MS] += \ + self.rank_bw_dict[f"{step}{Constant.STEP_RANK_SEP}{rank}"][self.SDMA_TIME_MS] += \ bw_dict.get(self.TRANSIT_TIME) if comm_type == self.RDMA: - self.rank_bw_dict[f"{step}{const.STEP_RANK_SEP}{rank}"][self.RDMA_SIZE_MB] += \ + self.rank_bw_dict[f"{step}{Constant.STEP_RANK_SEP}{rank}"][self.RDMA_SIZE_MB] += \ bw_dict.get(self.TRANSIT_SIZE) - self.rank_bw_dict[f"{step}{const.STEP_RANK_SEP}{rank}"][self.RDMA_TIME_MS] += \ + self.rank_bw_dict[f"{step}{Constant.STEP_RANK_SEP}{rank}"][self.RDMA_TIME_MS] += \ bw_dict.get(self.TRANSIT_TIME) for step_rank in self.rank_bw_dict.keys(): @@ -216,7 +215,7 @@ class ClusterCommunicationDataset(ClusterDataset): def _parse(self): self.cluster_analyze() try: - communication_json = self.load_json_data(const.CLUSTER_COMM_JSON) + communication_json = self.load_json_data(Constant.CLUSTER_COMM_JSON) except RuntimeError as e: logger.error("捕获到异常：%s", e) self.rank_bw_dict = None diff --git a/profiler/advisor/dataset/communication/communication_dataset.py b/profiler/advisor/dataset/communication/communication_dataset.py index 01a72ef9304..6a346f52ada 100644 --- a/profiler/advisor/dataset/communication/communication_dataset.py +++ b/profiler/advisor/dataset/communication/communication_dataset.py @@ -16,8 +16,8 @@ import logging import os from collections import defaultdict from profiler.advisor.utils.utils import singleton -from profiler.advisor.common import constant as const -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.constant import Constant +from profiler.prof_common.file_manager import FileManager from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo from profiler.advisor.utils.utils import CheckPathAccess @@ -34,7 +34,7 @@ class CommunicationDataset: return self.timeline_data_list = self.get_file_path_from_directory( self.timeline_dir, - lambda file: file.endswith(const.COMMUNICATION_JSON) + lambda file: file.endswith(Constant.COMMUNICATION_JSON) ) self.hccl_dict = defaultdict(list) self.step = kwargs.get("step") diff --git a/profiler/advisor/dataset/environment_variable_dataset.py b/profiler/advisor/dataset/environment_variable_dataset.py index 577273ffe8a..855b23189ea 100644 --- a/profiler/advisor/dataset/environment_variable_dataset.py +++ b/profiler/advisor/dataset/environment_variable_dataset.py @@ -15,8 +15,8 @@ import os import logging -from profiler.advisor.common import constant -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.constant import Constant +from profiler.prof_common.file_manager import FileManager class EnvironmentVariableDataset: @@ -29,7 +29,7 @@ class EnvironmentVariableDataset: def get_env_data_file(collection_path: str) -> str: for root, _, files in os.walk(collection_path): for file_name in files: - if file_name == constant.PROFILER_METADATA: + if file_name == Constant.PROFILER_METADATA: return os.path.join(root, file_name) return "" diff --git a/profiler/advisor/dataset/profiling/device_info.py b/profiler/advisor/dataset/profiling/device_info.py index abb0e6000c4..625e4ed0afa 100644 --- a/profiler/advisor/dataset/profiling/device_info.py +++ b/profiler/advisor/dataset/profiling/device_info.py @@ -6,7 +6,7 @@ import logging from profiler.advisor.config.config import Config from profiler.advisor.utils.utils import get_file_path_from_directory -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/advisor/dataset/profiling/profiling_dataset.py b/profiler/advisor/dataset/profiling/profiling_dataset.py index 0db673e1887..42847b88d2c 100644 --- a/profiler/advisor/dataset/profiling/profiling_dataset.py +++ b/profiler/advisor/dataset/profiling/profiling_dataset.py @@ -17,7 +17,7 @@ import logging import os import yaml -from profiler.advisor.common import constant +from profiler.prof_common.constant import Constant from profiler.advisor.common.profiling.ge_info import GeInfo from profiler.advisor.common.profiling.msprof import Msprof from profiler.advisor.common.profiling.op_summary import OpSummary @@ -26,7 +26,7 @@ from profiler.advisor.common.enum_params_parser import EnumParamsParser from profiler.advisor.dataset.dataset import Dataset from profiler.advisor.dataset.profiling.device_info import DeviceInfoParser from profiler.advisor.utils.utils import join_prof_path -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager logger = logging.getLogger() @@ -36,15 +36,15 @@ class ProfilingDataset(Dataset): prof_type = "" def __init__(self, collection_path, data: dict, **kwargs) -> None: - self.cann_version = kwargs.get(constant.CANN_VERSION, EnumParamsParser().get_default(constant.CANN_VERSION)) - self.prof_type = kwargs.get(constant.PROFILING_TYPE, EnumParamsParser().get_default(constant.PROFILING_TYPE)) + self.cann_version = kwargs.get(Constant.CANN_VERSION, EnumParamsParser().get_default(Constant.CANN_VERSION)) + self.prof_type = kwargs.get(Constant.PROFILING_TYPE_UNDER_LINE, EnumParamsParser().get_default(Constant.PROFILING_TYPE_UNDER_LINE)) self.patterns = self.parse_pattern() self.current_version_pattern = self.get_current_version_pattern() self._info = None super().__init__(collection_path, data) def build_from_pattern(self, dirs_pattern, current_path, depth): - if depth > constant.DEPTH_LIMIT: + if depth > Constant.DEPTH_LIMIT: logger.error("Recursion depth exceeds limit!") return depth += 1 diff --git a/profiler/advisor/dataset/profiling/profiling_parser.py b/profiler/advisor/dataset/profiling/profiling_parser.py index 70087c21827..73c9fc013ab 100644 --- a/profiler/advisor/dataset/profiling/profiling_parser.py +++ b/profiler/advisor/dataset/profiling/profiling_parser.py @@ -20,7 +20,7 @@ from typing import List, Dict from profiler.advisor.dataset.profiling.info_collection import logger from profiler.advisor.utils.utils import get_file_path_from_directory, SafeOpen, format_excel_title -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager class ProfilingParser: diff --git a/profiler/advisor/dataset/timeline_event_dataset.py b/profiler/advisor/dataset/timeline_event_dataset.py index 2cc6890f91f..fbac2c5bf01 100644 --- a/profiler/advisor/dataset/timeline_event_dataset.py +++ b/profiler/advisor/dataset/timeline_event_dataset.py @@ -21,7 +21,7 @@ from collections import OrderedDict import ijson from tqdm import tqdm -from profiler.advisor.common import constant as const +from profiler.prof_common.constant import Constant from profiler.advisor.common.timeline.event import TimelineEvent from profiler.advisor.utils.utils import get_file_path_from_directory, check_path_valid, singleton, convert_to_float from profiler.advisor.dataset.timeline_op_collector.timeline_op_collector import ( @@ -82,7 +82,7 @@ class BaseTimelineEventDataset: ops_with_task_type = getattr(self, "ops_with_task_type", {}).values() kwargs["ai_core_ops"] = [ op for op in ops_with_task_type if - op.get(const.TASK_TYPE) in [const.AI_CORE, const.MIX_AIC] + op.get(Constant.TASK_TYPE) in [Constant.AI_CORE, Constant.MIX_AIC] ] return kwargs @@ -201,7 +201,7 @@ class ScheduleAnalysisDataset(BaseTimelineEventDataset): return for event in sorted(self.aten, key=lambda x: x.get("ts", -1)): - if event.name.startswith(const.ATEN): + if event.name.startswith(Constant.ATEN): if not formated_atens or not formated_atens[-1].ts_include(event): formated_atens.append(event) diff --git a/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py b/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py index 824f4bc7bdd..f3350fb5255 100644 --- a/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py +++ b/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py @@ -3,10 +3,10 @@ import math import os from abc import abstractmethod, ABCMeta -from profiler.advisor.common import constant as const +from profiler.prof_common.constant import Constant from profiler.advisor.common.timeline.event import TimelineEvent from profiler.advisor.utils.utils import convert_to_float -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager logger = logging.getLogger() @@ -73,7 +73,7 @@ class OpCompileCollector(BaseOpCollector): self._total_op_compile_time = 0.0 def add_op(self, event): - if event.name == const.OP_COMPILE_NAME or event.args.get("id") == const.OP_COMPILE_ID: + if event.name == Constant.OP_COMPILE_NAME or event.args.get("id") == Constant.OP_COMPILE_ID: self.op_list.append(event) def post_process(self, target_op_list, **kwargs): @@ -90,7 +90,7 @@ class SynchronizeStreamCollector(BaseOpCollector): self.require_filter_by_step = False def add_op(self, event): - if event.name.startswith(const.SYNC_STREAM) or event.name.startswith(const.NODE_LAUNCH): + if event.name.startswith(Constant.SYNC_STREAM) or event.name.startswith(Constant.NODE_LAUNCH): self.op_list.append(event) def post_process(self, *args, **kwargs): @@ -170,13 +170,13 @@ class AtenCollector(BaseOpCollector): super().__init__() def add_op(self, event): - if event.name.lower().startswith(f"{const.ATEN}{const.ATEN_SEP}") or event.name.lower().startswith( - f"{const.NPU}{const.ATEN_SEP}"): + if event.name.lower().startswith(f"{Constant.ATEN}{Constant.ATEN_SEP}") or event.name.lower().startswith( + f"{Constant.NPU}{Constant.ATEN_SEP}"): self._add_aten(event) return # 检查cann层同步操作，根据时间窗口索引到host侧的aten算子并给出堆栈 - if event.name.startswith(const.SYNC_STREAM): + if event.name.startswith(Constant.SYNC_STREAM): self._add_aten(event) def post_process(self, target_op_list, **kwargs): @@ -194,7 +194,7 @@ class OptimizerCollector(BaseOpCollector): super().__init__() def add_op(self, event): - if event.name.startswith(f"{const.OPTIMIZER}.{const.OPTIMIZER_STEP}{const.OPTIMIZER_SEP}"): + if event.name.startswith(f"{Constant.OPTIMIZER}.{Constant.OPTIMIZER_STEP}{Constant.OPTIMIZER_SEP}"): self.op_list.append(TimelineEvent( {"name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur})) @@ -260,14 +260,14 @@ class SpecificTaskTypeOpCollector(BaseOpCollector): def __init__(self, op_type_list=None): super().__init__() - self.op_type_list = op_type_list if op_type_list else [const.AI_CPU, const.AI_CORE, const.MIX_AIC] + self.op_type_list = op_type_list if op_type_list else [Constant.AI_CPU, Constant.AI_CORE, Constant.MIX_AIC] def add_op(self, event): - if event.args.get(const.TASK_TYPE) and event.args.get(const.TASK_TYPE) in self.op_type_list: + if event.args.get(Constant.TASK_TYPE) and event.args.get(Constant.TASK_TYPE) in self.op_type_list: self.op_list.append( TimelineEvent( { - const.TASK_TYPE: event.args.get(const.TASK_TYPE), + Constant.TASK_TYPE: event.args.get(Constant.TASK_TYPE), "task_id": event.args.get("Task Id"), "tid": event.tid, "name": event.name, @@ -293,7 +293,7 @@ class TorchToNpuCollector(BaseOpCollector): super().__init__() def add_op(self, event): - if event.name.lower() == const.TORCH_TO_NPU: + if event.name.lower() == Constant.TORCH_TO_NPU: self.op_list.append(TimelineEvent({"tid": event.tid, "ts": str(event.ts), "ph": event.ph, "id": event.id})) def post_process(self, target_op_list, **kwargs): @@ -310,7 +310,7 @@ class AclToNpuCollector(BaseOpCollector): super().__init__() def add_op(self, event): - if event.name and event.ts and event.name == const.ACL_TO_NPU: + if event.name and event.ts and event.name == Constant.ACL_TO_NPU: self.op_list.append(TimelineEvent({"ts": event.ts})) def post_process(self, target_op_list, **kwargs): @@ -323,7 +323,7 @@ class OpStackCollector(BaseOpCollector): super().__init__() def add_op(self, event): - if event.args.get(const.CALL_STACKS): + if event.args.get(Constant.CALL_STACKS): self.op_list.append( TimelineEvent({"name": event.name, "dataset_index": event.dataset_index, "ts": event.ts})) @@ -363,13 +363,13 @@ class FreeEventsCollector(BaseOpCollector): return gc_rule def add_op(self, event): - if event.name.lower() == const.FREE: + if event.name.lower() == Constant.FREE: self.op_list.append(event) def post_process(self, target_op_list, **kwargs): gc_rule = self._load_rule() - if os.getenv(const.FREE_DURATION_FOR_GC_ANALYSIS): - max_free_threshold = convert_to_float(os.getenv(const.FREE_DURATION_FOR_GC_ANALYSIS)) + if os.getenv(Constant.FREE_DURATION_FOR_GC_ANALYSIS): + max_free_threshold = convert_to_float(os.getenv(Constant.FREE_DURATION_FOR_GC_ANALYSIS)) else: max_free_threshold = gc_rule.get("max_free_threshold") diff --git a/profiler/advisor/display/html/render.py b/profiler/advisor/display/html/render.py index d20df9a7601..804eacbaa20 100644 --- a/profiler/advisor/display/html/render.py +++ b/profiler/advisor/display/html/render.py @@ -19,7 +19,7 @@ from typing import List, Dict from collections import defaultdict, OrderedDict from jinja2 import Environment, FileSystemLoader -from profiler.advisor.common import constant +from profiler.prof_common.constant import Constant from profiler.advisor.config.config import Config from profiler.advisor.utils.utils import singleton, safe_write @@ -40,7 +40,7 @@ class HTMLRender: self.render_list = defaultdict(list) def render_html(self, template_dir: str = "templates", template_name: str = "main.html", - template_header=constant.DEFAULT_TEMPLATE_HEADER): + template_header=Constant.DEFAULT_TEMPLATE_HEADER): # 确保overall 和 comparison 在 performance problem analysis 之前 sorted_render_htmls = OrderedDict() diff --git a/profiler/advisor/result/result.py b/profiler/advisor/result/result.py index 0eadfd133a4..eb8e38c4108 100644 --- a/profiler/advisor/result/result.py +++ b/profiler/advisor/result/result.py @@ -22,11 +22,11 @@ import click import xlsxwriter from prettytable import ALL, PrettyTable -from profiler.advisor.common import constant as const +from profiler.prof_common.constant import Constant from profiler.advisor.utils.utils import singleton, logger from profiler.advisor.config.config import Config from profiler.advisor.utils.file import FdOpen, check_dir_writable -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager class ResultWriter: @@ -243,4 +243,4 @@ class TerminalResult: if table_row_cnt > 0: click.echo(self.table) else: - click.echo(click.style(const.SKIP_ANALYZE_PROMPT, fg='red')) + click.echo(click.style(Constant.SKIP_ANALYZE_PROMPT, fg='red')) diff --git a/profiler/advisor/utils/file.py b/profiler/advisor/utils/file.py index 767243bb48f..db01c389394 100644 --- a/profiler/advisor/utils/file.py +++ b/profiler/advisor/utils/file.py @@ -15,7 +15,7 @@ import os import logging -from profiler.advisor.common import constant as const +from profiler.prof_common.constant import Constant from profiler.advisor.utils.utils import check_path_valid from profiler.advisor.utils.log import get_log_level @@ -28,7 +28,7 @@ class FileOpen: open and read file """ - def __init__(self: any, file_path: str, mode: str = "r", max_size: int = const.MAX_READ_FILE_BYTES) -> None: + def __init__(self: any, file_path: str, mode: str = "r", max_size: int = Constant.MAX_READ_FILE_BYTES) -> None: self.file_path = file_path self.file_reader = None self.mode = mode @@ -49,7 +49,7 @@ class FdOpen: creat and write file """ - def __init__(self: any, file_path: str, flags: int = const.WRITE_FLAGS, mode: int = const.WRITE_MODES, + def __init__(self: any, file_path: str, flags: int = Constant.WRITE_FLAGS, mode: int = Constant.WRITE_MODES, operate: str = "w", newline: str = None) -> None: self.file_path = file_path self.flags = flags diff --git a/profiler/advisor/utils/log.py b/profiler/advisor/utils/log.py index b18272a82b6..5d9757ebf46 100644 --- a/profiler/advisor/utils/log.py +++ b/profiler/advisor/utils/log.py @@ -4,14 +4,14 @@ log module import logging import os -from profiler.advisor.common import constant as const +from profiler.prof_common.constant import Constant def get_log_level(): - log_level = os.getenv(const.ADVISOR_LOG_LEVEL, const.DEFAULT_LOG_LEVEL).upper() + log_level = os.getenv(Constant.ADVISOR_LOG_LEVEL, Constant.DEFAULT_LOG_LEVEL).upper() if not hasattr(logging, log_level): raise AttributeError(f"module 'logging' has no attribute '{log_level}', " - f"supported log level: {', '.join(const.SUPPORTED_LOG_LEVEL)}") + f"supported log level: {', '.join(Constant.SUPPORTED_LOG_LEVEL)}") return log_level diff --git a/profiler/advisor/utils/utils.py b/profiler/advisor/utils/utils.py index db8afd90465..7a78c6fa063 100644 --- a/profiler/advisor/utils/utils.py +++ b/profiler/advisor/utils/utils.py @@ -33,7 +33,7 @@ import requests from requests.adapters import HTTPAdapter from tqdm import tqdm -from profiler.advisor.common import constant as const +from profiler.prof_common.constant import Constant from profiler.advisor.common.version_control import VersionControl from profiler.advisor.utils.log import init_logger, get_log_level @@ -205,7 +205,7 @@ class Timer: def get_analyze_processes(): # n_processes not exposed to user through att-advisor command arguments now - return min(int(os.getenv(const.ADVISOR_ANALYZE_PROCESSES, 1)), const.ADVISOR_MAX_PROCESSES) + return min(int(os.getenv(Constant.ADVISOR_ANALYZE_PROCESSES, 1)), Constant.ADVISOR_MAX_PROCESSES) def format_timeline_result(result: dict, dump_html=False): @@ -439,7 +439,7 @@ def get_file_path_by_walk(root, filename): return file_path -def check_path_valid(path: str, is_file: bool = True, max_size: int = const.MAX_READ_FILE_BYTES) -> bool: +def check_path_valid(path: str, is_file: bool = True, max_size: int = Constant.MAX_READ_FILE_BYTES) -> bool: """ check the path is valid or not :param path: file path @@ -475,7 +475,7 @@ def parse_json_with_generator(timeline_data_path, func): return result try: with open(timeline_data_path, "r") as f: - if os.getenv(const.DISABLE_STREAMING_READER) == "1": + if os.getenv(Constant.DISABLE_STREAMING_READER) == "1": logger.debug("Disable streaming reader.") file_parser = json.loads(f.read()) else: diff --git a/profiler/cli/analyze_cli.py b/profiler/cli/analyze_cli.py index 456ecef07ae..c91c55c337a 100644 --- a/profiler/cli/analyze_cli.py +++ b/profiler/cli/analyze_cli.py @@ -8,7 +8,7 @@ sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__)), "cluste from profiler.advisor.analyzer.analyzer_controller import AnalyzerController from profiler.advisor.utils.tools import CONTEXT_SETTINGS, ClickAliasedGroup -from profiler.advisor.common import constant +from profiler.prof_common.constant import Constant from profiler.advisor.common.enum_params_parser import EnumParamsParser from profiler.advisor.utils.utils import debug_option from profiler.advisor.interface.interface import Interface @@ -33,20 +33,20 @@ def analyze_cli(**kwargs): @click.option('--output_path', '-o', 'output_path', type=click.Path(), help='Path of analysis output') @click.option('--cann_version', '-cv', 'cann_version', - type=click.Choice(EnumParamsParser().get_options(constant.CANN_VERSION), case_sensitive=False), - default=EnumParamsParser().get_default(constant.CANN_VERSION), + type=click.Choice(EnumParamsParser().get_options(Constant.CANN_VERSION), case_sensitive=False), + default=EnumParamsParser().get_default(Constant.CANN_VERSION), help='The CANN software version, which can be viewed by executing the following command: ' '"cat /usr/local/Ascend/ascend-toolkit/latest/aarch64-linux/ascend_toolkit_install.info"') @click.option('--torch_version', '-tv', 'torch_version', - type=click.Choice(EnumParamsParser().get_options(constant.TORCH_VERSION), case_sensitive=False), - default=EnumParamsParser().get_default(constant.TORCH_VERSION), + type=click.Choice(EnumParamsParser().get_options(Constant.TORCH_VERSION), case_sensitive=False), + default=EnumParamsParser().get_default(Constant.TORCH_VERSION), help='The runtime torch version, which can be detected by exec command "pip show torch"') @click.option("-pt", "--profiling_type", metavar="", - default=EnumParamsParser().get_default(constant.PROFILING_TYPE), + default=EnumParamsParser().get_default(Constant.PROFILING_TYPE_UNDER_LINE), required=False, - type=click.Choice(EnumParamsParser().get_options(constant.PROFILING_TYPE)), + type=click.Choice(EnumParamsParser().get_options(Constant.PROFILING_TYPE_UNDER_LINE)), help="enter the profiling type, selectable range ascend_pytorch_profiler, mslite ,msprof") @click.option("--force", is_flag=True, @@ -67,20 +67,20 @@ def analyze_all(**kwargs) -> None: @click.option('--output_path', '-o', 'output_path', type=click.Path(), help='Path of analysis output') @click.option('--cann_version', '-cv', 'cann_version', - type=click.Choice(EnumParamsParser().get_options(constant.CANN_VERSION), case_sensitive=False), - default=EnumParamsParser().get_default(constant.CANN_VERSION), + type=click.Choice(EnumParamsParser().get_options(Constant.CANN_VERSION), case_sensitive=False), + default=EnumParamsParser().get_default(Constant.CANN_VERSION), help='The CANN software version, which can be viewed by executing the following command: ' '"cat /usr/local/Ascend/ascend-toolkit/latest/aarch64-linux/ascend_toolkit_install.info"') @click.option('--torch_version', '-tv', 'torch_version', - type=click.Choice(EnumParamsParser().get_options(constant.TORCH_VERSION), case_sensitive=False), - default=EnumParamsParser().get_default(constant.TORCH_VERSION), + type=click.Choice(EnumParamsParser().get_options(Constant.TORCH_VERSION), case_sensitive=False), + default=EnumParamsParser().get_default(Constant.TORCH_VERSION), help='The runtime torch version, which can be detected by exec command "pip show torch"') @click.option("-pt", "--profiling_type", metavar="", - default=EnumParamsParser().get_default(constant.PROFILING_TYPE), + default=EnumParamsParser().get_default(Constant.PROFILING_TYPE_UNDER_LINE), required=False, - type=click.Choice(EnumParamsParser().get_options(constant.PROFILING_TYPE)), + type=click.Choice(EnumParamsParser().get_options(Constant.PROFILING_TYPE_UNDER_LINE)), help="enter the profiling type, selectable range ascend_pytorch_profiler, mslite ,msprof") @click.option("--force", is_flag=True, @@ -101,20 +101,20 @@ def analyze_schedule(**kwargs) -> None: @click.option('--output_path', '-o', 'output_path', type=click.Path(), help='Path of analysis output') @click.option('--cann_version', '-cv', 'cann_version', - type=click.Choice(EnumParamsParser().get_options(constant.CANN_VERSION), case_sensitive=False), - default=EnumParamsParser().get_default(constant.CANN_VERSION), + type=click.Choice(EnumParamsParser().get_options(Constant.CANN_VERSION), case_sensitive=False), + default=EnumParamsParser().get_default(Constant.CANN_VERSION), help='The CANN software version, which can be viewed by executing the following command: ' '"cat /usr/local/Ascend/ascend-toolkit/latest/aarch64-linux/ascend_toolkit_install.info"') @click.option('--torch_version', '-tv', 'torch_version', - type=click.Choice(EnumParamsParser().get_options(constant.TORCH_VERSION), case_sensitive=False), - default=EnumParamsParser().get_default(constant.TORCH_VERSION), + type=click.Choice(EnumParamsParser().get_options(Constant.TORCH_VERSION), case_sensitive=False), + default=EnumParamsParser().get_default(Constant.TORCH_VERSION), help='The runtime torch version, which can be detected by exec command "pip show torch"') @click.option("-pt", "--profiling_type", metavar="", - default=EnumParamsParser().get_default(constant.PROFILING_TYPE), + default=EnumParamsParser().get_default(Constant.PROFILING_TYPE_UNDER_LINE), required=False, - type=click.Choice(EnumParamsParser().get_options(constant.PROFILING_TYPE)), + type=click.Choice(EnumParamsParser().get_options(Constant.PROFILING_TYPE_UNDER_LINE)), help="enter the profiling type, selectable range ascend_pytorch_profiler, mslite ,msprof") @click.option("--force", is_flag=True, diff --git a/profiler/cluster_analyse/analysis/analysis_facade.py b/profiler/cluster_analyse/analysis/analysis_facade.py index 8d58666f3aa..4a8f50688a2 100644 --- a/profiler/cluster_analyse/analysis/analysis_facade.py +++ b/profiler/cluster_analyse/analysis/analysis_facade.py @@ -21,8 +21,7 @@ from analysis.comm_matrix_analysis import CommMatrixAnalysis from analysis.comm_matrix_analysis import CommMatrixAnalysisOptimized from analysis.step_trace_time_analysis import StepTraceTimeAnalysis from analysis.host_info_analysis import HostInfoAnalysis -from common_func.constant import Constant - +from profiler.prof_common.constant import Constant class AnalysisFacade: default_module = {CommunicationAnalysis, StepTraceTimeAnalysis, CommMatrixAnalysis, HostInfoAnalysis} diff --git a/profiler/cluster_analyse/analysis/base_analysis.py b/profiler/cluster_analyse/analysis/base_analysis.py index 60153d281a1..398098117a9 100644 --- a/profiler/cluster_analyse/analysis/base_analysis.py +++ b/profiler/cluster_analyse/analysis/base_analysis.py @@ -16,8 +16,8 @@ import logging from abc import abstractmethod -from common_func.constant import Constant -from common_func.file_manager import FileManager +from profiler.prof_common.constant import Constant +from profiler.prof_common.file_manager import FileManager from cluster_utils.data_transfer_adapter import DataTransferAdapter logger = logging.getLogger() diff --git a/profiler/cluster_analyse/analysis/comm_matrix_analysis.py b/profiler/cluster_analyse/analysis/comm_matrix_analysis.py index 4696f69438c..b4131b7611d 100644 --- a/profiler/cluster_analyse/analysis/comm_matrix_analysis.py +++ b/profiler/cluster_analyse/analysis/comm_matrix_analysis.py @@ -18,7 +18,7 @@ import logging from collections import defaultdict from analysis.base_analysis import BaseAnalysis -from common_func.constant import Constant +from profiler.prof_common.constant import Constant from common_func.db_manager import DBManager logger = logging.getLogger() diff --git a/profiler/cluster_analyse/analysis/communication_analysis.py b/profiler/cluster_analyse/analysis/communication_analysis.py index 68180ceee35..a46421193f6 100644 --- a/profiler/cluster_analyse/analysis/communication_analysis.py +++ b/profiler/cluster_analyse/analysis/communication_analysis.py @@ -18,11 +18,11 @@ import logging from collections import defaultdict from analysis.base_analysis import BaseAnalysis -from common_func.constant import Constant from common_func.table_constant import TableConstant from common_func.db_manager import DBManager from prof_bean.communication_bandwidth_bean import CommunicationBandwidthBean from prof_bean.communication_time_bean import CommunicationTimeBean +from profiler.prof_common.constant import Constant logger = logging.getLogger() diff --git a/profiler/cluster_analyse/analysis/host_info_analysis.py b/profiler/cluster_analyse/analysis/host_info_analysis.py index 871e314036e..f2d4ac9ffe1 100644 --- a/profiler/cluster_analyse/analysis/host_info_analysis.py +++ b/profiler/cluster_analyse/analysis/host_info_analysis.py @@ -17,7 +17,7 @@ import os import logging from analysis.base_analysis import BaseAnalysis -from common_func.constant import Constant +from profiler.prof_common.constant import Constant from common_func.db_manager import DBManager logger = logging.getLogger() diff --git a/profiler/cluster_analyse/analysis/step_trace_time_analysis.py b/profiler/cluster_analyse/analysis/step_trace_time_analysis.py index ce5648ce5bc..6fa1b62bf41 100644 --- a/profiler/cluster_analyse/analysis/step_trace_time_analysis.py +++ b/profiler/cluster_analyse/analysis/step_trace_time_analysis.py @@ -17,8 +17,8 @@ import os import logging from common_func.db_manager import DBManager -from common_func.constant import Constant -from common_func.file_manager import FileManager +from profiler.prof_common.constant import Constant +from profiler.prof_common.file_manager import FileManager from prof_bean.step_trace_time_bean import StepTraceTimeBean from cluster_utils.parallel_strategy_calculator import ParallelStrategyCalculator diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 3a4c8706175..4cce1ba3e65 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -17,16 +17,16 @@ import argparse import os import logging import sys -sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor from cluster_data_preprocess.mindspore_data_preprocessor import MindsporeDataPreprocessor from communication_group.communication_group_generator import CommunicationGroupGenerator -from common_func.constant import Constant -from common_func.file_manager import FileManager from common_func.path_manager import PathManager from analysis.analysis_facade import AnalysisFacade from profiler.prof_common.additional_args_manager import AdditionalArgsManager +from profiler.prof_common.constant import Constant +from profiler.prof_common.file_manager import FileManager COMM_FEATURE_LIST = ['all', 'communication_time', 'communication_matrix'] logger = logging.getLogger() diff --git a/profiler/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py b/profiler/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py index 573f82a5a47..bc6c8c12a97 100644 --- a/profiler/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py +++ b/profiler/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py @@ -18,8 +18,8 @@ from collections import defaultdict import os import logging from cluster_data_preprocess.data_preprocessor import DataPreprocessor -from common_func.constant import Constant -from common_func.file_manager import FileManager +from profiler.prof_common.constant import Constant +from profiler.prof_common.file_manager import FileManager class PytorchDataPreprocessor(DataPreprocessor): diff --git a/profiler/cluster_analyse/cluster_utils/data_transfer_adapter.py b/profiler/cluster_analyse/cluster_utils/data_transfer_adapter.py index 368eb6a9b21..c2aed8250de 100644 --- a/profiler/cluster_analyse/cluster_utils/data_transfer_adapter.py +++ b/profiler/cluster_analyse/cluster_utils/data_transfer_adapter.py @@ -1,6 +1,6 @@ import copy -from common_func.constant import Constant +from profiler.prof_common.constant import Constant from common_func.table_constant import TableConstant diff --git a/profiler/cluster_analyse/common_func/constant.py b/profiler/cluster_analyse/common_func/constant.py deleted file mode 100644 index f0486da9c3f..00000000000 --- a/profiler/cluster_analyse/common_func/constant.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright (c) 2023, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -class Constant(object): - # dir name - FRAMEWORK_DIR = "FRAMEWORK" - CLUSTER_ANALYSIS_OUTPUT = "cluster_analysis_output" - SINGLE_OUTPUT = "ASCEND_PROFILER_OUTPUT" - COMM_JSON = "communication.json" - COMM_MATRIX_JSON = "communication_matrix.json" - STEP_TIME_CSV = "step_trace_time.csv" - KERNEL_DETAILS_CSV = "kernel_details.csv" - - # file authority - FILE_AUTHORITY = 0o640 - DIR_AUTHORITY = 0o750 - MAX_JSON_SIZE = 1024 * 1024 * 1024 * 10 - MAX_CSV_SIZE = 1024 * 1024 * 1024 * 5 - MAX_PATH_LENGTH = 4096 - MAX_READ_DB_FILE_BYTES = 1024 * 1024 * 1024 * 8 - - # communication - P2P = "p2p" - COLLECTIVE = "collective" - STEP_ID = "step_id" - RANK_ID = "rank_id" - GROUP_NAME = "group_name" - COMM_OP_TYPE = "comm_op_type" - COMM_OP_NAME = "comm_op_name" - COMM_OP_INFO = "comm_op_info" - TOTAL_OP_INFO = "Total Op Info" - COMMUNICATION_TIME_INFO = "Communication Time Info" - START_TIMESTAMP = "Start Timestamp(us)" - COMMUNICATION_BANDWIDTH_INFO = "Communication Bandwidth Info" - HCOM_SEND = "hcom_send" - HCOM_RECEIVE = "hcom_receive" - SYNCHRONIZATION_TIME_RATIO = "Synchronization Time Ratio" - SYNCHRONIZATION_TIME_MS = "Synchronization Time(ms)" - WAIT_TIME_RATIO = "Wait Time Ratio" - TRANSIT_TIME_MS = "Transit Time(ms)" - TRANSIT_SIZE_MB = "Transit Size(MB)" - SIZE_DISTRIBUTION = "Size Distribution" - WAIT_TIME_MS = "Wait Time(ms)" - OP_NAME = "Op Name" - BANDWIDTH_GB_S = "Bandwidth(GB/s)" - COMMUNICATION = "communication.json" - ELAPSE_TIME_MS = "Elapse Time(ms)" - IDLE_TIME_MS = "Idle Time(ms)" - LARGE_PACKET_RATIO = "Large Packet Ratio" - - # params - DATA_MAP = "data_map" - COLLECTIVE_GROUP = "collective_group" - COMMUNICATION_OPS = "communication_ops" - MATRIX_OPS = "matrix_ops" - COLLECTION_PATH = "collection_path" - CLUSTER_ANALYSIS_OUTPUT_PATH = "output_path" - COMMUNICATION_GROUP = "communication_group" - TRANSPORT_TYPE = "Transport Type" - COMM_DATA_DICT = "comm_data_dict" - DATA_TYPE = "data_type" - ANALYSIS_MODE = "analysis_mode" - - # step time - RANK = "rank" - STAGE = "stage" - - # epsilon - EPS = 1e-15 - - # file suffix - JSON_SUFFIX = ".json" - CSV_SUFFIX = ".csv" - - # result files type - TEXT = "text" - DB = "db" - INVALID = "invalid" - - # db name - DB_COMMUNICATION_ANALYZER = "analysis.db" - DB_CLUSTER_COMMUNICATION_ANALYZER = "cluster_analysis.db" - - # db tables - TABLE_COMM_ANALYZER_BANDWIDTH = "CommAnalyzerBandwidth" - TABLE_COMM_ANALYZER_TIME = "CommAnalyzerTime" - TABLE_COMM_ANALYZER_MATRIX = "CommAnalyzerMatrix" - TABLE_STEP_TRACE = "StepTraceTime" - TABLE_HOST_INFO = "HostInfo" - TABLE_RANK_DEVICE_MAP = "RankDeviceMap" - - # data config key - CONFIG = "config" - EXPER_CONFIG = "experimental_config" - EXPORT_TYPE = "_export_type" - - # metadata key - DISTRIBUTED_ARGS = "distributed_args" - - # mode - ALL = "all" - COMMUNICATION_TIME = "communication_time" - COMMUNICATION_MATRIX = "communication_matrix" - - STEP = "step" - - DATA_SIMPLIFICATION = "data_simplification" - FORCE = "force" \ No newline at end of file diff --git a/profiler/cluster_analyse/common_func/db_manager.py b/profiler/cluster_analyse/common_func/db_manager.py index 1aa7ed8740e..85bcf975d65 100644 --- a/profiler/cluster_analyse/common_func/db_manager.py +++ b/profiler/cluster_analyse/common_func/db_manager.py @@ -16,9 +16,9 @@ import os import sqlite3 -from common_func.constant import Constant +from profiler.prof_common.constant import Constant from common_func.empty_class import EmptyClass -from common_func.file_manager import check_db_path_valid +from profiler.prof_common.file_manager import check_db_path_valid from common_func.tables_config import TablesConfig class DBManager: diff --git a/profiler/cluster_analyse/communication_group/base_communication_group.py b/profiler/cluster_analyse/communication_group/base_communication_group.py index d3a9d4eada9..2c44d0ae0f3 100644 --- a/profiler/cluster_analyse/communication_group/base_communication_group.py +++ b/profiler/cluster_analyse/communication_group/base_communication_group.py @@ -20,7 +20,7 @@ from copy import deepcopy from multiprocessing import Pool import logging -from common_func.constant import Constant +from profiler.prof_common.constant import Constant from cluster_utils.data_transfer_adapter import DataTransferAdapter diff --git a/profiler/cluster_analyse/communication_group/communication_db_group.py b/profiler/cluster_analyse/communication_group/communication_db_group.py index 37fea5a1c6d..eef617c9ef5 100644 --- a/profiler/cluster_analyse/communication_group/communication_db_group.py +++ b/profiler/cluster_analyse/communication_group/communication_db_group.py @@ -19,7 +19,7 @@ import os import logging from common_func.db_manager import DBManager -from common_func.constant import Constant +from profiler.prof_common.constant import Constant from communication_group.base_communication_group import BaseCommunicationGroup diff --git a/profiler/cluster_analyse/communication_group/communication_group_generator.py b/profiler/cluster_analyse/communication_group/communication_group_generator.py index f1e438977d1..324137207ca 100644 --- a/profiler/cluster_analyse/communication_group/communication_group_generator.py +++ b/profiler/cluster_analyse/communication_group/communication_group_generator.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from common_func.constant import Constant +from profiler.prof_common.constant import Constant from communication_group.communication_db_group import CommunicationDBGroup from communication_group.communication_db_group import CommunicationDBGroupOptimized from communication_group.communication_json_group import CommunicationJsonGroup diff --git a/profiler/cluster_analyse/communication_group/communication_json_group.py b/profiler/cluster_analyse/communication_group/communication_json_group.py index 6aaa5af6c9a..d9c1c0b4019 100644 --- a/profiler/cluster_analyse/communication_group/communication_json_group.py +++ b/profiler/cluster_analyse/communication_group/communication_json_group.py @@ -15,7 +15,7 @@ import os -from common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager from communication_group.base_communication_group import BaseCommunicationGroup diff --git a/profiler/compare_tools/compare_backend/comparator/api_compare_comparator.py b/profiler/compare_tools/compare_backend/comparator/api_compare_comparator.py index bc5810068b0..406614d1714 100644 --- a/profiler/compare_tools/compare_backend/comparator/api_compare_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/api_compare_comparator.py @@ -1,5 +1,5 @@ from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.utils.common_func import update_order_id diff --git a/profiler/compare_tools/compare_backend/comparator/communication_comparator.py b/profiler/compare_tools/compare_backend/comparator/communication_comparator.py index f7580bec89a..81342b6bfd8 100644 --- a/profiler/compare_tools/compare_backend/comparator/communication_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/communication_comparator.py @@ -1,6 +1,6 @@ from compare_backend.comparator.base_comparator import BaseComparator from compare_backend.compare_bean.communication_bean import CommunicationBean -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.utils.common_func import update_order_id diff --git a/profiler/compare_tools/compare_backend/comparator/kernel_compare_comparator.py b/profiler/compare_tools/compare_backend/comparator/kernel_compare_comparator.py index 13c0f776af6..ee26530db98 100644 --- a/profiler/compare_tools/compare_backend/comparator/kernel_compare_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/kernel_compare_comparator.py @@ -1,5 +1,5 @@ from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.utils.common_func import update_order_id diff --git a/profiler/compare_tools/compare_backend/comparator/module_comparetor.py b/profiler/compare_tools/compare_backend/comparator/module_comparetor.py index 49c50b53c5a..f1820e46eb7 100644 --- a/profiler/compare_tools/compare_backend/comparator/module_comparetor.py +++ b/profiler/compare_tools/compare_backend/comparator/module_comparetor.py @@ -1,6 +1,6 @@ from compare_backend.comparator.base_comparator import BaseComparator from compare_backend.utils.common_func import update_order_id -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant class ModuleComparator(BaseComparator): diff --git a/profiler/compare_tools/compare_backend/comparator/overall_metrics_comparator.py b/profiler/compare_tools/compare_backend/comparator/overall_metrics_comparator.py index d438dc41d56..4b9ded702b0 100644 --- a/profiler/compare_tools/compare_backend/comparator/overall_metrics_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/overall_metrics_comparator.py @@ -15,7 +15,7 @@ from math import isclose from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.utils.excel_config import ExcelConfig diff --git a/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py b/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py index 09d8688cf23..4ef3442c73a 100644 --- a/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py @@ -1,5 +1,5 @@ from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant class OverallPerformanceComparator(BaseComparator): diff --git a/profiler/compare_tools/compare_backend/compare_bean/api_compare_bean.py b/profiler/compare_tools/compare_backend/compare_bean/api_compare_bean.py index 8e83cf2817b..baad9013d43 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/api_compare_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/api_compare_bean.py @@ -14,7 +14,7 @@ # limitations under the License. from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.utils.excel_config import ExcelConfig diff --git a/profiler/compare_tools/compare_backend/compare_bean/communication_bean.py b/profiler/compare_tools/compare_backend/compare_bean/communication_bean.py index e6afce2b538..6ea7c0eb0c0 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/communication_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/communication_bean.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.utils.excel_config import ExcelConfig from compare_backend.utils.common_func import calculate_diff_ratio diff --git a/profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py b/profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py index 44b761a4e05..528713d60f1 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py @@ -14,7 +14,7 @@ # limitations under the License. from compare_backend.utils.common_func import calculate_diff_ratio, convert_to_float -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.utils.excel_config import ExcelConfig diff --git a/profiler/compare_tools/compare_backend/compare_bean/memory_compare_bean.py b/profiler/compare_tools/compare_backend/compare_bean/memory_compare_bean.py index d345ec90f0d..7892271840f 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/memory_compare_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/memory_compare_bean.py @@ -14,7 +14,7 @@ # limitations under the License. from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.utils.excel_config import ExcelConfig from compare_backend.utils.torch_op_node import TorchOpNode from compare_backend.utils.tree_builder import TreeBuilder diff --git a/profiler/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py b/profiler/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py index b7b3d4a97a2..ab6119fbd94 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py @@ -14,7 +14,7 @@ # limitations under the License. from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.utils.tree_builder import TreeBuilder from compare_backend.utils.excel_config import ExcelConfig diff --git a/profiler/compare_tools/compare_backend/compare_bean/module_compare_bean.py b/profiler/compare_tools/compare_backend/compare_bean/module_compare_bean.py index 0d979d8a700..f490232b39f 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/module_compare_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/module_compare_bean.py @@ -14,7 +14,7 @@ # limitations under the License. from compare_backend.utils.common_func import longest_common_subsequence_matching, calculate_diff_ratio -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.utils.excel_config import ExcelConfig from compare_backend.utils.module_node import ModuleNode from compare_backend.utils.name_function import NameFunction diff --git a/profiler/compare_tools/compare_backend/compare_bean/module_statistic_bean.py b/profiler/compare_tools/compare_backend/compare_bean/module_statistic_bean.py index cd9a2062a06..2d607b4639c 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/module_statistic_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/module_statistic_bean.py @@ -16,7 +16,7 @@ import re from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.utils.excel_config import ExcelConfig diff --git a/profiler/compare_tools/compare_backend/compare_bean/operator_compare_bean.py b/profiler/compare_tools/compare_backend/compare_bean/operator_compare_bean.py index aaa1e785347..e8782a1e6ff 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/operator_compare_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/operator_compare_bean.py @@ -14,7 +14,7 @@ # limitations under the License. from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.utils.excel_config import ExcelConfig from compare_backend.utils.torch_op_node import TorchOpNode from compare_backend.utils.tree_builder import TreeBuilder diff --git a/profiler/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py b/profiler/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py index e79c8ab32b5..a81542272fc 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py @@ -14,7 +14,7 @@ # limitations under the License. from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.utils.excel_config import ExcelConfig from compare_backend.utils.tree_builder import TreeBuilder diff --git a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py b/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py index 9de6726ed11..08a515ad267 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py +++ b/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py @@ -16,7 +16,7 @@ from decimal import Decimal from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant class KernelEvent: diff --git a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py b/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py index 04be6866e84..3b4f4b3d556 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py @@ -19,8 +19,8 @@ from decimal import Decimal import pandas as pd from compare_backend.utils.common_func import convert_to_float, convert_to_decimal -from compare_backend.utils.constant import Constant from compare_backend.compare_config.compare_config import CompareConfig +from profiler.prof_common.constant import Constant class KernelDetailsBean: diff --git a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py b/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py index a4c914a4556..4677e79a63b 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py @@ -17,8 +17,8 @@ from decimal import Decimal from compare_backend.utils.common_func import convert_to_float, convert_to_decimal -from compare_backend.utils.constant import Constant from compare_backend.compare_config.compare_config import CompareConfig +from profiler.prof_common.constant import Constant class TraceEventBean: diff --git a/profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py b/profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py index b818850d44e..88666cf7082 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py @@ -16,7 +16,7 @@ from math import isclose from compare_backend.compare_bean.profiling_info import ProfilingInfo from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.utils.excel_config import ExcelConfig, CellFormatType diff --git a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py index 2089a139838..559415cb8d5 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py +++ b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant class ProfilingInfo: diff --git a/profiler/compare_tools/compare_backend/comparison_generator.py b/profiler/compare_tools/compare_backend/comparison_generator.py index 5c3c82dd855..8f8fb15f519 100644 --- a/profiler/compare_tools/compare_backend/comparison_generator.py +++ b/profiler/compare_tools/compare_backend/comparison_generator.py @@ -22,7 +22,7 @@ from compare_backend.interface.overall_interface import OverallInterface from compare_backend.interface.compare_interface import CompareInterface from compare_backend.profiling_parser.gpu_profiling_parser import GPUProfilingParser from compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingParser -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.utils.args_manager import ArgsManager from profiler.prof_common.additional_args_manager import AdditionalArgsManager diff --git a/profiler/compare_tools/compare_backend/data_prepare/module_data_prepare.py b/profiler/compare_tools/compare_backend/data_prepare/module_data_prepare.py index 116de8fb6df..ccbfc8ff8c5 100644 --- a/profiler/compare_tools/compare_backend/data_prepare/module_data_prepare.py +++ b/profiler/compare_tools/compare_backend/data_prepare/module_data_prepare.py @@ -17,7 +17,7 @@ from queue import Queue from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean from compare_backend.profiling_parser.base_profiling_parser import ProfilingResult -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.utils.module_node import ModuleNode from compare_backend.utils.tree_builder import TreeBuilder diff --git a/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py b/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py index d5fb3990a94..fdca9009f9d 100644 --- a/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py +++ b/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py @@ -15,7 +15,7 @@ import logging from compare_backend.profiling_parser.base_profiling_parser import ProfilingResult from compare_backend.utils.tree_builder import TreeBuilder -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant class OperatorDataPrepare: diff --git a/profiler/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py b/profiler/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py index c04d4c2b699..364e5e531bf 100644 --- a/profiler/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py +++ b/profiler/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py @@ -19,7 +19,7 @@ from compare_backend.utils.common_func import longest_common_subsequence_matchin from compare_backend.utils.torch_op_node import TorchOpNode from compare_backend.utils.module_node import ModuleNode -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant class SequencePreMatching: diff --git a/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py b/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py index 1b0b75300d8..fa7b6126f6c 100644 --- a/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py +++ b/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py @@ -18,7 +18,7 @@ from compare_backend.profiling_parser.gpu_profiling_parser import GPUProfilingPa from compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingParser from compare_backend.utils.args_manager import ArgsManager from compare_backend.utils.compare_args import Args -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant class OverallPerfInterface: diff --git a/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py b/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py index e7ab32e07b9..fc46453536f 100644 --- a/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py +++ b/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py @@ -37,7 +37,7 @@ from compare_backend.compare_bean.overall_metrics_bean import OverallMetricsBean from compare_backend.data_prepare.module_data_prepare import ModuleDataPrepare from compare_backend.data_prepare.operator_data_prepare import OperatorDataPrepare from compare_backend.generator.base_generator import BaseGenerator -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.view.excel_view import ExcelView from compare_backend.data_prepare.sequence_pre_matching import SequencePreMatching diff --git a/profiler/compare_tools/compare_backend/interface/compare_interface.py b/profiler/compare_tools/compare_backend/interface/compare_interface.py index 67c5db67fb4..67426b99c0b 100644 --- a/profiler/compare_tools/compare_backend/interface/compare_interface.py +++ b/profiler/compare_tools/compare_backend/interface/compare_interface.py @@ -1,4 +1,4 @@ -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant from compare_backend.comparator.operator_comparator import OperatorComparator from compare_backend.comparator.api_compare_comparator import ApiCompareComparator @@ -7,7 +7,6 @@ from compare_backend.compare_bean.operator_compare_bean import OperatorCompareBe from compare_backend.compare_bean.api_compare_bean import ApiCompareBean from compare_backend.compare_bean.kernel_compare_bean import KernelCompareBean from compare_backend.data_prepare.operator_data_prepare import OperatorDataPrepare -from compare_backend.utils.constant import Constant from compare_backend.data_prepare.sequence_pre_matching import SequencePreMatching class CompareInterface: diff --git a/profiler/compare_tools/compare_backend/interface/overall_interface.py b/profiler/compare_tools/compare_backend/interface/overall_interface.py index fb549007f63..5ccde19ca18 100644 --- a/profiler/compare_tools/compare_backend/interface/overall_interface.py +++ b/profiler/compare_tools/compare_backend/interface/overall_interface.py @@ -1,6 +1,6 @@ from compare_backend.comparator.overall_performance_comparator import OverallPerformanceComparator from compare_backend.compare_bean.profiling_info import ProfilingInfo -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant class OverallInterface: diff --git a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py index 2e5df19827d..59a307e4681 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py @@ -6,8 +6,8 @@ from compare_backend.compare_bean.origin_data_bean.compare_event import KernelEv from compare_backend.compare_bean.origin_data_bean.kernel_details_bean import KernelDetailsBean from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean from compare_backend.compare_bean.profiling_info import ProfilingInfo -from compare_backend.utils.constant import Constant -from compare_backend.utils.file_reader import FileReader +from profiler.prof_common.constant import Constant +from profiler.prof_common.file_manager import FileManager logger = logging.getLogger() @@ -333,6 +333,6 @@ class BaseProfilingParser(ABC): def _read_trace_event(self): try: - self._trace_events = FileReader.read_trace_file(self._json_path) + self._trace_events = FileManager.read_json_file(self._json_path) except Exception: print(f"[ERROR] Failed to read the file: {self._json_path}") diff --git a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index c2e5a148056..3f2a8ee600b 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -3,7 +3,7 @@ from collections import defaultdict, Counter from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean from compare_backend.profiling_parser.base_profiling_parser import BaseProfilingParser -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant class GPUProfilingParser(BaseProfilingParser): diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 515085ec8f0..839178bb9ae 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -8,8 +8,8 @@ from compare_backend.compare_bean.origin_data_bean.memory_record_bean import Mem from compare_backend.compare_bean.origin_data_bean.operator_memory_bean import OperatorMemoryBean from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean from compare_backend.profiling_parser.base_profiling_parser import BaseProfilingParser -from compare_backend.utils.constant import Constant -from compare_backend.utils.file_reader import FileReader +from profiler.prof_common.constant import Constant +from profiler.prof_common.file_manager import FileManager logger = logging.getLogger() @@ -66,7 +66,7 @@ class NPUProfilingParser(BaseProfilingParser): data = [] file_name = os.path.basename(file_path) try: - data = FileReader.read_csv_file(file_path, bean) + data = FileManager.read_csv_file(file_path, bean) except FileNotFoundError: logger.warning("The file %s does not exist.", file_name) except Exception: @@ -164,7 +164,7 @@ class NPUProfilingParser(BaseProfilingParser): def _update_bandwidth(self): try: - communication_json = FileReader.read_trace_file(self._communication_path) + communication_json = FileManager.read_json_file(self._communication_path) except FileNotFoundError: logger.warning("The file communication.json does not exist.") return @@ -327,7 +327,7 @@ class NPUProfilingParser(BaseProfilingParser): def __parse_info_json(self): try: - json_data = FileReader.read_trace_file(self._info_json_path) + json_data = FileManager.read_json_file(self._info_json_path) except Exception: logger.error('Failed to read profiler_info.json.') return @@ -349,7 +349,7 @@ class NPUProfilingParser(BaseProfilingParser): def __parse_kernel_csv(self): try: - kernel_details = FileReader.read_csv_file(self._kernel_detail_path, KernelDetailsBean) + kernel_details = FileManager.read_csv_file(self._kernel_detail_path, KernelDetailsBean) except Exception: logger.error('Npu kernel details csv file is not available.') return @@ -365,7 +365,7 @@ class NPUProfilingParser(BaseProfilingParser): def __parse_mem_csv(self): try: - memory_record = FileReader.read_csv_file(self._memory_record_path, MemoryRecordBean) + memory_record = FileManager.read_csv_file(self._memory_record_path, MemoryRecordBean) except FileNotFoundError: logger.warning('Npu memory record csv file is not available.') except Exception: diff --git a/profiler/compare_tools/compare_backend/utils/args_manager.py b/profiler/compare_tools/compare_backend/utils/args_manager.py index f18d3b51449..aca2974d2b1 100644 --- a/profiler/compare_tools/compare_backend/utils/args_manager.py +++ b/profiler/compare_tools/compare_backend/utils/args_manager.py @@ -19,10 +19,9 @@ import os.path import re from common_func.path_manager import PathManager -from compare_backend.utils.constant import Constant -from compare_backend.utils.file_reader import FileReader from compare_backend.utils.singleton import Singleton - +from profiler.prof_common.constant import Constant +from profiler.prof_common.file_manager import FileManager @Singleton class ArgsManager: @@ -125,7 +124,7 @@ class ArgsManager: if extension != ".json": msg = f"Invalid profiling path suffix: {file_path}" raise RuntimeError(msg) - json_type = FileReader.check_json_type(file_path) + json_type = FileManager.check_json_type(file_path) return { Constant.PROFILING_TYPE: json_type, Constant.PROFILING_PATH: file_path, Constant.TRACE_PATH: file_path } diff --git a/profiler/compare_tools/compare_backend/utils/constant.py b/profiler/compare_tools/compare_backend/utils/constant.py deleted file mode 100644 index e06363a4886..00000000000 --- a/profiler/compare_tools/compare_backend/utils/constant.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - - -class Constant(object): - GPU = "GPU" - NPU = "NPU" - NA = 'N/A' - LIMIT_KERNEL = 3 - MAX_PATH_LENGTH = 4096 - MAX_FLOW_CAT_LEN = 20 - MAX_OP_NAME_LEN = 200 - MAX_FILE_SIZE = 1024 * 1024 * 1024 * 5 - MAX_JSON_SIZE = 1024 * 1024 * 1024 * 10 - BYTE_TO_KB = 1024 - YELLOW_COLOR = "FFFF00" - GREEN_COLOR = "00FF00" - RED_COLOR = "FF0000" - BLUE_COLOR = "00BFFF" - LIGHT_BLUE_COLOR = "87CEFA" - US_TO_MS = 1000 - KB_TO_MB = 1024 - INVALID_VALUE = -1 - MILLISECONDS_TO_SECONDS = 10 ** 3 - MICROSECONDS_TO_SECONDS = 10 ** 6 - - # epsilon - EPS = 1e-15 - - # autority - FILE_AUTHORITY = 0o640 - DIR_AUTHORITY = 0o750 - - PROFILING_TYPE = "profiling type" - - # path - PROFILING_PATH = "profiling_path" - TRACE_PATH = "trace_path" - MEMORY_DATA_PATH = "memory_data_path" - ASCEND_OUTPUT_PATH = "ascend_output" - INFO_JSON_PATH = "info_path" - - # excel headers - BASE_PROFILING = 'Base Profiling: ' - COMPARISON_PROFILING = 'Comparison Profiling: ' - WAIT_TIME = "wait" - TRANSMIT_TIME = "transmit" - - # compare type - OPERATOR_COMPARE = "OperatorCompare" - MEMORY_COMPARE = "MemoryCompare" - API_COMPARE = "ApiCompare" - KERNEL_COMPARE = "KernelCompare" - - # sheet name - OPERATOR_SHEET = "OperatorCompare" - MEMORY_SHEET = "MemoryCompare" - OPERATOR_TOP_SHEET = "OperatorCompareStatistic" - MEMORY_TOP_SHEET = "MemoryCompareStatistic" - COMMUNICATION_SHEET = "CommunicationCompare" - API_SHEET = "ApiCompare" - KERNEL_SHEET = "KernelCompare" - - # table name - OPERATOR_TABLE = "OperatorCompare" - MEMORY_TABLE = "MemoryCompare" - OPERATOR_TOP_TABLE = "OperatorCompareStatistic" - MEMORY_TOP_TABLE = "MemoryCompareStatistic" - COMMUNICATION_TABLE = "CommunicationCompare" - PERFORMANCE_TABLE = "Model Profiling Time Distribution" - MODULE_TABLE = "ModuleCompare" - MODULE_TOP_TABLE = "ModuleCompareStatistic" - OVERALL_METRICS_TABLE = "OverallMetrics" - API_TABLE = "ApiCompare" - KERNEL_TABLE = "KernelCompare" - - # memory - SIZE = "Size(KB)" - TS = "ts" - ALLOCATION_TIME = "Allocation Time(us)" - RELEASE_TIME = "Release Time(us)" - NAME = "Name" - - OP_KEY = "op_name" - DEVICE_DUR = "dur" - - BASE_DATA = "base_data" - COMPARISON_DATA = "comparison_data" - OVERALL_METRICS = "overall_metrics" - TORCH_OP = "torch_op" - KERNEL_DICT = "kernel_dict" - MEMORY_LIST = "memory_list" - COMMUNICATION_DICT = "comm_dict" - - # compare type - OVERALL_COMPARE = "overall" - - BWD_LIST = ["bwd", "backward", "back", "grad"] - IS_BWD = "is_bwd" - OPS = "ops" - - VOID_STEP = -1 diff --git a/profiler/compare_tools/compare_backend/utils/excel_config.py b/profiler/compare_tools/compare_backend/utils/excel_config.py index fb12f8a67c0..899a20b6a46 100644 --- a/profiler/compare_tools/compare_backend/utils/excel_config.py +++ b/profiler/compare_tools/compare_backend/utils/excel_config.py @@ -15,7 +15,7 @@ # limitations under the License. """ -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant class CellFormatType: diff --git a/profiler/compare_tools/compare_backend/utils/file_reader.py b/profiler/compare_tools/compare_backend/utils/file_reader.py deleted file mode 100644 index acdad9a33cb..00000000000 --- a/profiler/compare_tools/compare_backend/utils/file_reader.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import csv -import json -import os -import logging - -from common_func.path_manager import PathManager -from compare_backend.utils.constant import Constant -from profiler.prof_common.additional_args_manager import AdditionalArgsManager - - -logger = logging.getLogger() - - -class FileReader: - @classmethod - def read_trace_file(cls, file_path: str) -> any: - if not os.path.isfile(file_path): - raise FileNotFoundError("File not exists.") - PathManager.check_path_readable(file_path) - file_size = os.path.getsize(file_path) - if file_size <= 0: - return [] - if not AdditionalArgsManager().force and file_size > Constant.MAX_FILE_SIZE: - check_msg = input( - f"The file({file_path}) size exceeds the preset max value. Continue reading the file? [y/n]") - if check_msg.lower() != "y": - logger.warning("The user choose not to read the file: %s", file_path) - return [] - try: - with open(file_path, "rt") as file: - json_data = json.loads(file.read()) - except Exception as e: - msg = f"Can't read file: {file_path}" - raise RuntimeError(msg) from e - return json_data - - @classmethod - def read_csv_file(cls, file_path: str, bean_class: any = None) -> any: - if not os.path.isfile(file_path): - raise FileNotFoundError("File not exists.") - PathManager.check_path_readable(file_path) - file_size = os.path.getsize(file_path) - if file_size <= 0: - return [] - if not AdditionalArgsManager().force and file_size > Constant.MAX_FILE_SIZE: - check_msg = input( - f"The file({file_path}) size exceeds the preset max value. Continue reading the file? [y/n]") - if check_msg.lower() != "y": - print(f"[WARNING] The user choose not to read the file: {file_path}") - return [] - result_data = [] - try: - with open(file_path, newline="") as csv_file: - reader = csv.DictReader(csv_file) - for row in reader: - row_data = bean_class(row) if bean_class else row - result_data.append(row_data) - except Exception as e: - msg = f"Failed to read the file: {file_path}" - raise RuntimeError(msg) from e - return result_data - - @classmethod - def check_json_type(cls, file_path: str) -> str: - json_data = cls.read_trace_file(file_path) - if isinstance(json_data, dict): - return Constant.GPU - return Constant.NPU diff --git a/profiler/compare_tools/compare_backend/utils/torch_op_node.py b/profiler/compare_tools/compare_backend/utils/torch_op_node.py index 693f55c4a66..75d2c79a304 100644 --- a/profiler/compare_tools/compare_backend/utils/torch_op_node.py +++ b/profiler/compare_tools/compare_backend/utils/torch_op_node.py @@ -17,7 +17,7 @@ from compare_backend.compare_bean.origin_data_bean.compare_event import MemoryEvent from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant class TorchOpNode: diff --git a/profiler/compare_tools/compare_backend/view/excel_view.py b/profiler/compare_tools/compare_backend/view/excel_view.py index 73b82b1cd31..f16a10e31b7 100644 --- a/profiler/compare_tools/compare_backend/view/excel_view.py +++ b/profiler/compare_tools/compare_backend/view/excel_view.py @@ -4,7 +4,7 @@ from xlsxwriter import Workbook from compare_backend.view.base_view import BaseView from compare_backend.view.work_sheet_creator import WorkSheetCreator -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant class ExcelView(BaseView): diff --git a/profiler/compare_tools/compare_interface/comparison_interface.py b/profiler/compare_tools/compare_interface/comparison_interface.py index 936e5a7e8e7..0d6e66a230a 100644 --- a/profiler/compare_tools/compare_interface/comparison_interface.py +++ b/profiler/compare_tools/compare_interface/comparison_interface.py @@ -8,7 +8,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from compare_backend.comparison_generator import ComparisonGenerator from compare_backend.disaggregate.overall_perf_interface import OverallPerfInterface from compare_backend.utils.compare_args import Args -from compare_backend.utils.constant import Constant +from profiler.prof_common.constant import Constant class ComparisonInterface: diff --git a/profiler/prof_common/constant.py b/profiler/prof_common/constant.py index 4d3d1ce6344..b49ab5f2369 100644 --- a/profiler/prof_common/constant.py +++ b/profiler/prof_common/constant.py @@ -14,6 +14,10 @@ # limitations under the License. + +import os +import stat + class Constant(object): COLLECTION_PATH = "collection_path" ANALYSIS_MODE = "analysis_mode" @@ -31,3 +35,348 @@ class Constant(object): FWD_OR_OPT = 0 BACKWARD = 1 INVALID_RETURN = -1 + + # dir name + FRAMEWORK_DIR = "FRAMEWORK" + CLUSTER_ANALYSIS_OUTPUT = "cluster_analysis_output" + SINGLE_OUTPUT = "ASCEND_PROFILER_OUTPUT" + COMM_JSON = "communication.json" + COMM_MATRIX_JSON = "communication_matrix.json" + STEP_TIME_CSV = "step_trace_time.csv" + KERNEL_DETAILS_CSV = "kernel_details.csv" + + # file authority + FILE_AUTHORITY = 0o640 + DIR_AUTHORITY = 0o750 + MAX_JSON_SIZE = 1024 * 1024 * 1024 * 10 + MAX_CSV_SIZE = 1024 * 1024 * 1024 * 5 + MAX_TRACE_SIZE = 1024 * 1024 * 1024 * 5 + MAX_PATH_LENGTH = 4096 + MAX_READ_DB_FILE_BYTES = 1024 * 1024 * 1024 * 8 + + # communication + P2P = "p2p" + COLLECTIVE = "collective" + STEP_ID = "step_id" + RANK_ID = "rank_id" + GROUP_NAME = "group_name" + COMM_OP_TYPE = "comm_op_type" + COMM_OP_NAME = "comm_op_name" + COMM_OP_INFO = "comm_op_info" + TOTAL_OP_INFO = "Total Op Info" + COMMUNICATION_TIME_INFO = "Communication Time Info" + START_TIMESTAMP = "Start Timestamp(us)" + COMMUNICATION_BANDWIDTH_INFO = "Communication Bandwidth Info" + HCOM_SEND = "hcom_send" + HCOM_RECEIVE = "hcom_receive" + SYNCHRONIZATION_TIME_RATIO = "Synchronization Time Ratio" + SYNCHRONIZATION_TIME_MS = "Synchronization Time(ms)" + WAIT_TIME_RATIO = "Wait Time Ratio" + TRANSIT_TIME_MS = "Transit Time(ms)" + TRANSIT_SIZE_MB = "Transit Size(MB)" + SIZE_DISTRIBUTION = "Size Distribution" + WAIT_TIME_MS = "Wait Time(ms)" + OP_NAME = "Op Name" + BANDWIDTH_GB_S = "Bandwidth(GB/s)" + COMMUNICATION = "communication.json" + ELAPSE_TIME_MS = "Elapse Time(ms)" + IDLE_TIME_MS = "Idle Time(ms)" + LARGE_PACKET_RATIO = "Large Packet Ratio" + + # params + DATA_MAP = "data_map" + COLLECTIVE_GROUP = "collective_group" + COMMUNICATION_OPS = "communication_ops" + MATRIX_OPS = "matrix_ops" + CLUSTER_ANALYSIS_OUTPUT_PATH = "output_path" + COMMUNICATION_GROUP = "communication_group" + TRANSPORT_TYPE = "Transport Type" + COMM_DATA_DICT = "comm_data_dict" + DATA_TYPE = "data_type" + + # step time + RANK = "rank" + STAGE = "stage" + + # epsilon + EPS = 1e-15 + + # file suffix + JSON_SUFFIX = ".json" + CSV_SUFFIX = ".csv" + + # result files type + TEXT = "text" + DB = "db" + INVALID = "invalid" + + # db name + DB_COMMUNICATION_ANALYZER = "analysis.db" + DB_CLUSTER_COMMUNICATION_ANALYZER = "cluster_analysis.db" + + # db tables + TABLE_COMM_ANALYZER_BANDWIDTH = "CommAnalyzerBandwidth" + TABLE_COMM_ANALYZER_TIME = "CommAnalyzerTime" + TABLE_COMM_ANALYZER_MATRIX = "CommAnalyzerMatrix" + TABLE_STEP_TRACE = "StepTraceTime" + TABLE_HOST_INFO = "HostInfo" + TABLE_RANK_DEVICE_MAP = "RankDeviceMap" + + # data config key + CONFIG = "config" + EXPER_CONFIG = "experimental_config" + EXPORT_TYPE = "_export_type" + + # metadata key + DISTRIBUTED_ARGS = "distributed_args" + + # mode + ALL = "all" + COMMUNICATION_TIME = "communication_time" + COMMUNICATION_MATRIX = "communication_matrix" + + STEP = "step" + + DATA_SIMPLIFICATION = "data_simplification" + FORCE = "force" + + # compare tools + + GPU = "GPU" + NPU = "NPU" + NA = 'N/A' + LIMIT_KERNEL = 3 + MAX_PATH_LENGTH = 4096 + MAX_FLOW_CAT_LEN = 20 + MAX_OP_NAME_LEN = 200 + MAX_FILE_SIZE = 1024 * 1024 * 1024 * 5 + BYTE_TO_KB = 1024 + YELLOW_COLOR = "FFFF00" + GREEN_COLOR = "00FF00" + RED_COLOR = "FF0000" + BLUE_COLOR = "00BFFF" + LIGHT_BLUE_COLOR = "87CEFA" + US_TO_MS = 1000 + KB_TO_MB = 1024 + INVALID_VALUE = -1 + MILLISECONDS_TO_SECONDS = 10 ** 3 + MICROSECONDS_TO_SECONDS = 10 ** 6 + + PROFILING_TYPE = "profiling type" + + # path + PROFILING_PATH = "profiling_path" + TRACE_PATH = "trace_path" + MEMORY_DATA_PATH = "memory_data_path" + ASCEND_OUTPUT_PATH = "ascend_output" + INFO_JSON_PATH = "info_path" + + # excel headers + BASE_PROFILING = 'Base Profiling: ' + COMPARISON_PROFILING = 'Comparison Profiling: ' + WAIT_TIME = "wait" + TRANSMIT_TIME = "transmit" + + # compare type + OPERATOR_COMPARE = "OperatorCompare" + MEMORY_COMPARE = "MemoryCompare" + API_COMPARE = "ApiCompare" + KERNEL_COMPARE = "KernelCompare" + + # sheet name + OPERATOR_SHEET = "OperatorCompare" + MEMORY_SHEET = "MemoryCompare" + OPERATOR_TOP_SHEET = "OperatorCompareStatistic" + MEMORY_TOP_SHEET = "MemoryCompareStatistic" + COMMUNICATION_SHEET = "CommunicationCompare" + API_SHEET = "ApiCompare" + KERNEL_SHEET = "KernelCompare" + + # table name + OPERATOR_TABLE = "OperatorCompare" + MEMORY_TABLE = "MemoryCompare" + OPERATOR_TOP_TABLE = "OperatorCompareStatistic" + MEMORY_TOP_TABLE = "MemoryCompareStatistic" + COMMUNICATION_TABLE = "CommunicationCompare" + PERFORMANCE_TABLE = "Model Profiling Time Distribution" + MODULE_TABLE = "ModuleCompare" + MODULE_TOP_TABLE = "ModuleCompareStatistic" + OVERALL_METRICS_TABLE = "OverallMetrics" + API_TABLE = "ApiCompare" + KERNEL_TABLE = "KernelCompare" + + # memory + SIZE = "Size(KB)" + TS = "ts" + ALLOCATION_TIME = "Allocation Time(us)" + RELEASE_TIME = "Release Time(us)" + NAME = "Name" + + OP_KEY = "op_name" + DEVICE_DUR = "dur" + + BASE_DATA = "base_data" + COMPARISON_DATA = "comparison_data" + OVERALL_METRICS = "overall_metrics" + TORCH_OP = "torch_op" + KERNEL_DICT = "kernel_dict" + MEMORY_LIST = "memory_list" + COMMUNICATION_DICT = "comm_dict" + + # compare type + OVERALL_COMPARE = "overall" + + BWD_LIST = ["bwd", "backward", "back", "grad"] + + CPU_OP_FA_MASK = ( + "flash_attention", "fusion_attention", "flashattn", "xformers_flash", "efficient_attention", "flash2attn" + ) + CPU_OP_CONV = "aten::conv" + CPU_OP_MATMUL_MASK = ("aten::addmm", "aten::bmm", "aten::mm", "aten::matmul") + KERNEL_CUBE_MASK = ("gemm", "conv", "cutlass", "wgrad", "gemvx") + KERNEL_TRANS_MASK = ("cast", "transdata", "transpose") + + IS_BWD = "is_bwd" + OPS = "ops" + + VOID_STEP = -1 + + # advisor + + # timeline + DEQUEUE = "Dequeue" + DEQUEUE_SEP = "@" + ATEN = "aten" + NPU = "npu" + ATEN_SEP = "::" + OPTIMIZER = "Optimizer" + OPTIMIZER_SEP = "#" + OPTIMIZER_STEP = "step" + ENQUEUE = "enqueue" + TORCH_TO_NPU = "torch_to_npu" + FREE = "free" + OP_COMPILE_NAME = "AscendCL@aclopCompileAndExecute" + OP_COMPILE_ID = "aclopCompileAndExecute" + SYNC_STREAM = "AscendCL@aclrtSynchronizeStream" + NODE_LAUNCH = "Node@launch" + MAX_OP_COMPILE_NUM = 20 + ACL_TO_NPU = "acl_to_npu" + TASK_TYPE = "Task Type" + CPU_OP = "cpu_op" + AI_CORE = "AI_CORE" + AI_CPU = "AI_CPU" + MIX_AIC = "MIX_AIC" + CALL_STACKS = "Call stack" + INPUT_DIMS = "Input Dims" + OP_SEP = "-" + ADVISOR_MAX_PROCESSES = 8 + ADVISOR_ANALYZE_PROCESSES = "ADVISOR_ANALYZE_PROCESSES" + TIMELINE_OP_STACKS_DATASET = "timeline_op_stacks_dataset" + TIMELINE_BACKWARD_NO_STACK = "Backward broadcast, without call stacks in profiling." + TIMELINE_ACL_TO_NPU_NO_STACK = "Incoming flow is 'acl_to_npu', without call stacks in profiling." + TIMELINE_BACKWARD_NO_STACK_CODE = -1 + TIMELINE_ACL_TO_NPU_NO_STACK_CODE = -2 + TIMELINE_FUSION_OPS_NO_STACK_FLAG = "NO STACK" + NO_STACK_REASON_MAP = { + TIMELINE_BACKWARD_NO_STACK_CODE: "Backward broadcast, without call stacks in profiling.", + TIMELINE_ACL_TO_NPU_NO_STACK_CODE: "Incoming flow is 'acl_to_npu', without call stacks in profiling." + } + AFFINITY_TRAINING_API = "Affinity training api" + TIMELINE_EMPTY_STACKS_PROMPT = "These APIs have no code stack. If parameter 'with_stack=False' while profiling, " \ + "please refer to {timeline_profiling_doc_url} to set 'with_stack=True'. " \ + "Otherwise, ignore following affinity APIs due to backward broadcast lack of stack." + + CLUSTER_ANALYSIS = "Cluster analysis" + SLOW_RANK_TIME_RATIO_THRESHOLD = 0.05 + + CANN_VERSION = "cann_version" + TORCH_VERSION = "torch_version" + PROFILING_TYPE_UNDER_LINE = "profiling_type" + ANALYSIS_DIMENSIONS = "analysis_dimensions" + + PROFILER_METADATA = "profiler_metadata.json" + + TERMINAL_OUTPUT_HEADERS = ["No.", "Problem", "Description", "Suggestion"] + SKIP_ANALYZE_PROMPT = "Finish analysis, no optimization suggestions" + SKIP_QUERY_PROMPT = "Finish query operator stack, no operators" + + # operator output constant + OPERATOR_OUT_TOPK = 10 + OPERATOR_LIST_UNLIMIT = -1 + + DEFAULT_OPERATOR_TYPE = 'None_type' + DEFAULT_DURATION_ZERO = 0.0 + + ADVISOR_LOG_LEVEL = "ADVISOR_LOG_LEVEL" + DEFAULT_LOG_LEVEL = "INFO" + SUPPORTED_LOG_LEVEL = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] + + RULE_BUCKET = "RULE-BUCKET" + CLOUD_RULE_REGION_CN_NORTH_9 = "cn-north-9" + CLOUD_RULE_REGION_CN_NORTH_7 = "cn-north-7" + CLOUD_RULE_REGION_CN_SOUTHWEST_2 = "cn-southwest-2" + CLOUD_RULE_REGION_LIST = [CLOUD_RULE_REGION_CN_NORTH_7, CLOUD_RULE_REGION_CN_NORTH_9, CLOUD_RULE_REGION_CN_SOUTHWEST_2] + INNER_REGION_LIST = [CLOUD_RULE_REGION_CN_NORTH_7] + DEFAULT_CLOUD_RULE_REGION = CLOUD_RULE_REGION_CN_SOUTHWEST_2 + + HTTP_PREFIXES = "http://" + HTTPS_PREFIXES = "https://" + COMMON_YAML_DIR = "modelarts/solution/ma_advisor_rules/" + COMMON_ENDPOINT_SUFFIX = "obs.{}.myhuaweicloud.com" + INNER_ENDPOINT_SUFFIX = "obs.{}.ulanqab.huawei.com" + + AICPU_RULES_YAML_NAME = "aicpu_rules.yaml" + FUSION_PASS_YAML_NAME = "op_fusion_pass.yaml" + TIMELINE_FUSION_OPS_YAML_NAME = "timeline_fusion_ops.yaml" + CLOUD_YAML_NAME_LIST = [AICPU_RULES_YAML_NAME, FUSION_PASS_YAML_NAME, TIMELINE_FUSION_OPS_YAML_NAME] + + MAX_RETRIES = 3 + TIMEOUT = 3 + DEPTH_LIMIT = 20 + + ADVISOR_RULE_PATH = "ADVISOR_RULE_PATH" + CLOUD_RULE_PATH = "rules/cloud/" + DEFAULT_RULE_PATH = "./rules/" + + TIMELINE_FUSION_OPS_INVALID_UNIQUE_ID = -1 + + DEFAULT_TEMPLATE_HEADER = "Performance Optimization Suggestions" + + PT_PROF_SUFFIX = "ascend_pt" + ASCEND_PROFILER_OUTPUT = "ASCEND_PROFILER_OUTPUT" + KERNEL_DETAILS_CSV = "kernel_details.csv" + CLUSTER_STEP_TIME_CSV = "cluster_step_trace_time.csv" + CLUSTER_COMM_JSON = "cluster_communication.json" + COMMUNICATION_JSON = "communication.json" + + BOTTLENECK = "bottleneck" + DATA = "data" + ADVISOR_ANALYSIS_OUTPUT_DIR = "advisor_analysis_result" + DEFAULT_PROCESSES = 8 + CLUSTER_ANALYSIS_FILE_PATTERN = [ + r'profiler_info_\d+\.json', "step_trace_time.csv", "communication.json", "communication_matrix.json" + ] + ANALYSIS_OUTPUT_PATH = "ANALYSIS_OUTPUT_PATH" + DEFAULT_RANK_FOR_PROFILING_ANALYSIS = 0 + PROFILER_INFO_FILE_PATTERN = r"profiler_info_(\d+)\.json" + DISABLE_STREAMINIG_READER = "DISABLE_STREAMINIG_READER" + FRAMEWORK_STACK_BLACK_LIST = ["torch", "torch_npu", "megatron", "deepspeed"] + DISABLE_STREAMING_READER = "DISABLE_STREAMING_READER" + MAX_NUM_PROCESSES = 4 + DEFAULT_STEP = "-1" + STEP_RANK_SEP = "_" + + MAX_READ_LINE_BYTES = 8196 * 1024 + MAX_READ_FILE_BYTES = 64 * 1024 * 1024 * 1024 + MAX_READ_DB_FILE_BYTES = 8 * 1024 * 1024 * 1024 + + # Unit Conversion + COMMUNICATION_B_TO_GB = 0.001 ** 3 + US_TO_S = 0.001 ** 2 + + WRITE_MODES = stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP + WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC + + DISABLE_PROFILING_COMPARISON = "DISABLE_PROFILING_COMPARISON" + FREE_DURATION_FOR_GC_ANALYSIS = "FREE_DURATION_FOR_GC_ANALYSIS" + DISABLE_AFFINITY_API = "DISABLE_AFFINITY_API" \ No newline at end of file diff --git a/profiler/cluster_analyse/common_func/file_manager.py b/profiler/prof_common/file_manager.py similarity index 78% rename from profiler/cluster_analyse/common_func/file_manager.py rename to profiler/prof_common/file_manager.py index 96ea823706a..10bc51c6083 100644 --- a/profiler/cluster_analyse/common_func/file_manager.py +++ b/profiler/prof_common/file_manager.py @@ -20,52 +20,69 @@ import logging import yaml -from common_func.constant import Constant -from common_func.path_manager import PathManager +from profiler.prof_common.constant import Constant +from profiler.prof_common.path_manager import PathManager from profiler.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() class FileManager: - DATA_FILE_AUTHORITY = 0o640 - DATA_DIR_AUTHORITY = 0o750 + @classmethod - def read_csv_file(cls, file_path: str, class_bean: any) -> list: + def read_json_file(cls, file_path: str) -> dict: PathManager.check_path_readable(file_path) base_name = os.path.basename(file_path) file_size = os.path.getsize(file_path) if file_size <= 0: - return [] - if file_size > Constant.MAX_CSV_SIZE and not AdditionalArgsManager().force: - raise RuntimeError(f"The file({base_name}) size exceeds the preset max value.") - result_data = [] + return {} + if not AdditionalArgsManager().force and file_size > Constant.MAX_FILE_SIZE: + check_msg = input( + f"The file({file_path}) size exceeds the preset max value. Continue reading the file? [y/n]") + if check_msg.lower() != "y": + logger.warning("The user choose not to read the file: %s", file_path) + return [] try: - with open(file_path, newline="") as csv_file: - reader = csv.DictReader(csv_file) - for row in reader: - result_data.append(class_bean(row)) + with open(file_path, "r") as json_file: + result_data = json.loads(json_file.read()) except Exception as e: raise RuntimeError(f"Failed to read the file: {base_name}") from e return result_data @classmethod - def read_json_file(cls, file_path: str) -> dict: + def read_csv_file(cls, file_path: str, class_bean: any = None) -> list: + if not os.path.isfile(file_path): + raise FileNotFoundError("File not exists.") PathManager.check_path_readable(file_path) - base_name = os.path.basename(file_path) file_size = os.path.getsize(file_path) if file_size <= 0: - return {} - if file_size > Constant.MAX_JSON_SIZE and not AdditionalArgsManager().force: - raise RuntimeError(f"The file({base_name}) size exceeds the preset max value.") + return [] + if not AdditionalArgsManager().force and file_size > Constant.MAX_FILE_SIZE: + check_msg = input( + f"The file({file_path}) size exceeds the preset max value. Continue reading the file? [y/n]") + if check_msg.lower() != "y": + print(f"[WARNING] The user choose not to read the file: {file_path}") + return [] + result_data = [] try: - with open(file_path, "r") as json_file: - result_data = json.loads(json_file.read()) + with open(file_path, newline="") as csv_file: + reader = csv.DictReader(csv_file) + for row in reader: + row_data = class_bean(row) if class_bean else row + result_data.append(row_data) except Exception as e: - raise RuntimeError(f"Failed to read the file: {base_name}") from e + msg = f"Failed to read the file: {file_path}" + raise RuntimeError(msg) from e return result_data + @classmethod + def check_json_type(cls, file_path: str) -> str: + json_data = cls.read_json_file(file_path) + if isinstance(json_data, dict): + return Constant.GPU + return Constant.NPU + @classmethod def read_yaml_file(cls, file_path: str) -> dict: PathManager.check_path_readable(file_path) @@ -94,7 +111,7 @@ class FileManager: PathManager.check_path_writeable(output_path) try: with os.fdopen( - os.open(output_file, os.O_WRONLY | os.O_CREAT, cls.DATA_FILE_AUTHORITY), + os.open(output_file, os.O_WRONLY | os.O_CREAT, Constant.FILE_AUTHORITY), 'w', newline="" ) as file: writer = csv.writer(file) @@ -118,7 +135,7 @@ class FileManager: base_name = os.path.basename(output_file) try: with os.fdopen( - os.open(output_file, os.O_WRONLY | os.O_CREAT, cls.DATA_FILE_AUTHORITY), 'w' + os.open(output_file, os.O_WRONLY | os.O_CREAT, Constant.FILE_AUTHORITY), 'w' ) as file: file.write(json.dumps(data)) except Exception as e: diff --git a/profiler/test/run_st.py b/profiler/test/run_st.py index d0437e7a4e9..bf33e0f3e5e 100644 --- a/profiler/test/run_st.py +++ b/profiler/test/run_st.py @@ -7,7 +7,7 @@ import threading stop_print_thread = False - +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) def print_stout(output): while True: line = output.readline().strip() diff --git a/profiler/test/run_ut.py b/profiler/test/run_ut.py index 6ab208dc29e..0fa5340b79e 100644 --- a/profiler/test/run_ut.py +++ b/profiler/test/run_ut.py @@ -2,7 +2,7 @@ import os import shutil import subprocess import sys - +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) def set_python_path(): cluster_analyse_root = os.path.join( diff --git a/profiler/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py b/profiler/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py index a6c62dca674..a2f2fcff8d6 100644 --- a/profiler/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py +++ b/profiler/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py @@ -12,7 +12,7 @@ from advisor_backend.cluster_advice.cluster_pipeline_advice import PipelineTrace class TestClusterPipelineAdvice(unittest.TestCase): def test_load_trace_view_data_should_return_none_when_input_json_empty(self): - with mock.patch('common_func.file_manager.FileManager.read_json_file', return_value=None): + with mock.patch('profiler.prof_common.file_manager.FileManager.read_json_file', return_value=None): advice = ClusterPipelineAdvice('./tmp_dir', {}) self.assertEqual(advice.load_trace_view_data('test'), None) @@ -62,7 +62,7 @@ class TestClusterPipelineAdvice(unittest.TestCase): npu_ops_ts_dur={"15": 16, "17": 18}, torch_to_npu_links=[torch_to_npu_link], ) - with mock.patch('common_func.file_manager.FileManager.read_json_file', return_value=raw_data): + with mock.patch('profiler.prof_common.file_manager.FileManager.read_json_file', return_value=raw_data): advice = ClusterPipelineAdvice('./tmp_dir', {}) check_res = advice.load_trace_view_data('test') self.assertEqual(check_res, except_res) diff --git a/profiler/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py b/profiler/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py index 0509b197caf..e7f0cb85bcb 100644 --- a/profiler/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py +++ b/profiler/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py @@ -5,7 +5,7 @@ import unittest from unittest import mock from unittest.mock import MagicMock -from common_func.constant import Constant +from profiler.prof_common.constant import Constant from advisor_backend.cluster_advice.kernel_cluster_advice import KernelClusterAdvice diff --git a/profiler/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py b/profiler/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py index 7b141ae0886..144561c4178 100644 --- a/profiler/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py +++ b/profiler/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py @@ -5,7 +5,6 @@ import unittest from unittest import mock from unittest.mock import MagicMock -from common_func.constant import Constant from advisor_backend.prof_bean_advisor.cluster_step_trace_time_bean import ClusterStepTraceTimeBean diff --git a/profiler/test/ut/advisor/timeline_advice/test_synchronize_stream.py b/profiler/test/ut/advisor/timeline_advice/test_synchronize_stream.py index 674d149640d..8f5d3d4cae8 100644 --- a/profiler/test/ut/advisor/timeline_advice/test_synchronize_stream.py +++ b/profiler/test/ut/advisor/timeline_advice/test_synchronize_stream.py @@ -5,7 +5,7 @@ import yaml from profiler.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_checker import SynchronizeStreamChecker from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.common import constant as const +from profiler.prof_common.constant import Constant from profiler.advisor.utils.utils import safe_division from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env @@ -49,13 +49,13 @@ class TestSynchronizeChecker(unittest.TestCase): if is_empty_dataset: return dataset - co_occurrence_event_list = [TimelineEvent(dict(name=const.NODE_LAUNCH)), - TimelineEvent(dict(name=const.SYNC_STREAM))] * co_occurrence_num + co_occurrence_event_list = [TimelineEvent(dict(name=Constant.NODE_LAUNCH)), + TimelineEvent(dict(name=Constant.SYNC_STREAM))] * co_occurrence_num - synchronize_stream_event_list = [TimelineEvent(dict(name=const.SYNC_STREAM))] * ( + synchronize_stream_event_list = [TimelineEvent(dict(name=Constant.SYNC_STREAM))] * ( total_synchronize_stream_num - co_occurrence_num) - node_launch_event_list = [TimelineEvent(dict(name=const.NODE_LAUNCH))] * ( + node_launch_event_list = [TimelineEvent(dict(name=Constant.NODE_LAUNCH))] * ( total_node_launch_num - co_occurrence_num) dataset[ diff --git a/profiler/test/ut/advisor/timeline_advice/test_timeline_op_compile_checker.py b/profiler/test/ut/advisor/timeline_advice/test_timeline_op_compile_checker.py index 3ebdcddad3d..e83c57e0a2f 100644 --- a/profiler/test/ut/advisor/timeline_advice/test_timeline_op_compile_checker.py +++ b/profiler/test/ut/advisor/timeline_advice/test_timeline_op_compile_checker.py @@ -18,7 +18,7 @@ class TestOperatorDispatchAnalyzer(unittest.TestCase): def tearDownClass(cls) -> None: recover_env() - @patch("profiler.advisor.common.constant.MAX_OP_COMPILE_NUM", 5) + @patch("profiler.prof_common.constant.Constant.MAX_OP_COMPILE_NUM", 5) def test_ops_dispatch_analyzer(self): kwargs = {"analysis_mode": "all"} data_root_dir = os.path.dirname(os.path.realpath(__file__)) @@ -28,7 +28,7 @@ class TestOperatorDispatchAnalyzer(unittest.TestCase): self.assertTrue(results.page_dict) self.assertIsNotNone(results.sheet_recorder.sheet_data.get("operator dispatch")) - @patch("profiler.advisor.common.constant.MAX_OP_COMPILE_NUM", 5) + @patch("profiler.prof_common.constant.Constant.MAX_OP_COMPILE_NUM", 5) def test_ops_dispatch_make_render(self): kwargs = {"analysis_mode": "timeline"} data_root_dir = os.path.dirname(os.path.realpath(__file__)) diff --git a/profiler/test/ut/cluster_analyse/common_func/test_file_manager.py b/profiler/test/ut/cluster_analyse/common_func/test_file_manager.py index 5f73b20244e..071d11e7ffb 100644 --- a/profiler/test/ut/cluster_analyse/common_func/test_file_manager.py +++ b/profiler/test/ut/cluster_analyse/common_func/test_file_manager.py @@ -5,7 +5,7 @@ import json import unittest import pytest -from common_func.file_manager import FileManager +from profiler.prof_common.file_manager import FileManager from prof_bean.step_trace_time_bean import StepTraceTimeBean diff --git a/profiler/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py b/profiler/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py index 3d9ff4512d4..690a5e80c54 100644 --- a/profiler/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py +++ b/profiler/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py @@ -38,7 +38,7 @@ class TestNPUProfilingParser(unittest.TestCase): with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", return_value=None), \ - patch("compare_backend.utils.file_reader.FileReader.read_csv_file", return_value=memory_data): + patch("profiler.prof_common.file_manager.FileManager.read_csv_file", return_value=memory_data): res = NPUProfilingParser({}, {}) res._operator_memory_path = "" res._enqueue_dict = {} diff --git a/profiler/test/ut/compare_tools/utils/test_file_reader.py b/profiler/test/ut/compare_tools/utils/test_file_reader.py deleted file mode 100644 index de7e13f9e53..00000000000 --- a/profiler/test/ut/compare_tools/utils/test_file_reader.py +++ /dev/null @@ -1,19 +0,0 @@ -import unittest - -from compare_backend.utils.file_reader import FileReader -from compare_backend.utils.constant import Constant - - -class TestFileReader(unittest.TestCase): - - def test_read_trace_file(self): - json_data = FileReader.read_trace_file("resource/event_list.json") - self.assertEqual(len(json_data), 2) - - def test_read_csv_file(self): - csv = FileReader.read_csv_file("resource/test.csv") - self.assertEqual(len(csv), 8) - - def test_check_json_type(self): - t = FileReader.check_json_type("resource/event_list.json") - self.assertEqual(t, Constant.NPU) -- Gitee From 9b1a4c539c0dabd7ef6474ddab502232fafc297b Mon Sep 17 00:00:00 2001 From: stby <295887736@qq.com> Date: Thu, 21 Nov 2024 17:10:01 +0800 Subject: [PATCH 2/2] test for path manager --- .../timeline_op_collector.py | 2 +- profiler/cluster_analyse/cluster_analysis.py | 2 +- .../cluster_prof_Info_analysis.py | 2 +- profiler/prof_common/__init__.py | 3 +++ profiler/prof_common/constant.py | 2 +- profiler/prof_common/path_manager.py | 13 +++++++++++-- profiler/test/run_st.py | 1 - profiler/test/run_ut.py | 1 - ...est_advisor_cmd_single_ascend_pt_no_compare.py | 15 ++++++++------- .../common_func/test_path_manager.py | 2 +- 10 files changed, 27 insertions(+), 16 deletions(-) diff --git a/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py b/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py index f3350fb5255..52f15caef19 100644 --- a/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py +++ b/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py @@ -171,7 +171,7 @@ class AtenCollector(BaseOpCollector): def add_op(self, event): if event.name.lower().startswith(f"{Constant.ATEN}{Constant.ATEN_SEP}") or event.name.lower().startswith( - f"{Constant.NPU}{Constant.ATEN_SEP}"): + f"{Constant.NPU_LOWER}{Constant.ATEN_SEP}"): self._add_aten(event) return diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 4cce1ba3e65..6e749f0f08c 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -22,11 +22,11 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath( from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor from cluster_data_preprocess.mindspore_data_preprocessor import MindsporeDataPreprocessor from communication_group.communication_group_generator import CommunicationGroupGenerator -from common_func.path_manager import PathManager from analysis.analysis_facade import AnalysisFacade from profiler.prof_common.additional_args_manager import AdditionalArgsManager from profiler.prof_common.constant import Constant from profiler.prof_common.file_manager import FileManager +from profiler.prof_common.path_manager import PathManager COMM_FEATURE_LIST = ['all', 'communication_time', 'communication_matrix'] logger = logging.getLogger() diff --git a/profiler/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py b/profiler/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py index 4cc49eb6411..3af835e6db2 100644 --- a/profiler/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py +++ b/profiler/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py @@ -29,7 +29,7 @@ from plotly.offline import plot sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from common_func.path_manager import PathManager +from profiler.prof_common.path_manager import PathManager from profiler.prof_common.additional_args_manager import AdditionalArgsManager diff --git a/profiler/prof_common/__init__.py b/profiler/prof_common/__init__.py index e69de29bb2d..9ee8a1b9b09 100644 --- a/profiler/prof_common/__init__.py +++ b/profiler/prof_common/__init__.py @@ -0,0 +1,3 @@ +import os +import sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) \ No newline at end of file diff --git a/profiler/prof_common/constant.py b/profiler/prof_common/constant.py index b49ab5f2369..86c86a8fe52 100644 --- a/profiler/prof_common/constant.py +++ b/profiler/prof_common/constant.py @@ -247,7 +247,7 @@ class Constant(object): DEQUEUE = "Dequeue" DEQUEUE_SEP = "@" ATEN = "aten" - NPU = "npu" + NPU_LOWER = "npu" ATEN_SEP = "::" OPTIMIZER = "Optimizer" OPTIMIZER_SEP = "#" diff --git a/profiler/prof_common/path_manager.py b/profiler/prof_common/path_manager.py index e6942c2c42e..68f2d63b6e6 100644 --- a/profiler/prof_common/path_manager.py +++ b/profiler/prof_common/path_manager.py @@ -17,8 +17,8 @@ import re import shutil import platform -from .constant import Constant -from .additional_args_manager import AdditionalArgsManager +from profiler.prof_common.constant import Constant +from profiler.prof_common.additional_args_manager import AdditionalArgsManager class PathManager: @@ -88,6 +88,13 @@ class PathManager: msg = f"Invalid path which has illagal characters \"{invalid_obj}\"." raise RuntimeError(msg) + path_split_list = path.split("/") + for path in path_split_list: + path_list = path.split("\\") + for name in path_list: + if len(name) > cls.MAX_FILE_NAME_LENGTH: + raise RuntimeError("Length of input path exceeds the limit.") + @classmethod def check_path_owner_consistent(cls, path_list: list): """ @@ -147,6 +154,8 @@ class PathManager: @classmethod def remove_path_safety(cls, path: str): + if not os.path.exists(path): + return base_name = os.path.basename(path) msg = f"Failed to remove path: {base_name}" cls.check_path_writeable(path) diff --git a/profiler/test/run_st.py b/profiler/test/run_st.py index bf33e0f3e5e..f7c689f5f1b 100644 --- a/profiler/test/run_st.py +++ b/profiler/test/run_st.py @@ -7,7 +7,6 @@ import threading stop_print_thread = False -sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) def print_stout(output): while True: line = output.readline().strip() diff --git a/profiler/test/run_ut.py b/profiler/test/run_ut.py index 0fa5340b79e..e75af167ff1 100644 --- a/profiler/test/run_ut.py +++ b/profiler/test/run_ut.py @@ -2,7 +2,6 @@ import os import shutil import subprocess import sys -sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) def set_python_path(): cluster_analyse_root = os.path.join( diff --git a/profiler/test/st/advisor/test_advisor_cmd_single_ascend_pt_no_compare.py b/profiler/test/st/advisor/test_advisor_cmd_single_ascend_pt_no_compare.py index e243f465681..ce6eeb3a39c 100644 --- a/profiler/test/st/advisor/test_advisor_cmd_single_ascend_pt_no_compare.py +++ b/profiler/test/st/advisor/test_advisor_cmd_single_ascend_pt_no_compare.py @@ -50,6 +50,7 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): def test_all_problems(self): category = [ "overall summary", + "byte alignment analysis", "bandwidth contention analysis", "AICPU operator", "Dynamic shape operator", @@ -58,13 +59,13 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): ] #True presents the attr is nan - description_len = [6,3,2,1,1,1] - suggestion_len = [True,1,2,5,1,1] - problem_count = [True,True,2.0,1.0,True,True] - total_time = [True,True,57674709.54,True,True,True] - time_ratio = [True,True,0.0,True,True,True] - income = [True,True,True,True,True,True] - income_ratio = [True,True,True,True,True,True] + description_len = [6,1,3,2,1,1,1] + suggestion_len = [True,1,1,2,5,1,1] + problem_count = [True,True,True,2.0,1.0,True,True] + total_time = [True,True,True,57674709.54,True,True,True] + time_ratio = [True,True,True,0.0,True,True,True] + income = [True,True,True,True,True,True,True] + income_ratio = [True,True,True,True,True,True,True] try: df = pd.read_excel(self.RESULT_EXCEL.get("all",None), sheet_name='problems',header=0) except FileNotFoundError: diff --git a/profiler/test/ut/cluster_analyse/common_func/test_path_manager.py b/profiler/test/ut/cluster_analyse/common_func/test_path_manager.py index 16fd340b2a7..93b5f3e1f3e 100644 --- a/profiler/test/ut/cluster_analyse/common_func/test_path_manager.py +++ b/profiler/test/ut/cluster_analyse/common_func/test_path_manager.py @@ -3,7 +3,7 @@ import os import time import pytest -from common_func.path_manager import PathManager +from profiler.prof_common.path_manager import PathManager PATH_DIR = "resource" -- Gitee