From 067ef7b4a17b83752b5be0fc4821bc5a0f6b1b10 Mon Sep 17 00:00:00 2001 From: hehongzhe <935062458@qq.com> Date: Sun, 26 Jan 2025 23:08:57 +0800 Subject: [PATCH 1/7] =?UTF-8?q?import=E6=95=B4=E6=94=B9=20advisor=20affini?= =?UTF-8?q?ty=5Fcpu=5Fbind=20cli?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 10 +++++----- .../advice_factory/cluster_advice_factory.py | 12 +++++------ .../advice_factory/compute_advice_factory.py | 8 ++++---- .../advice_factory/overall_advice_factory.py | 6 +++--- .../advice_factory/timeline_advice_factory.py | 8 ++++---- .../cluster_advice/cluster_advice_base.py | 5 ++--- .../cluster_advice/cluster_pipeline_advice.py | 9 ++++----- .../cluster_advice/kernel_cluster_advice.py | 4 ++-- .../cluster_advice/slow_link_advice.py | 4 ++-- .../cluster_advice/slow_rank_advice.py | 6 +++--- .../common_func_advisor/trace_view_json.py | 2 -- .../trace_view_preprocessor.py | 2 +- .../compute_advice/compute_advice_base.py | 2 +- .../compute_advice/npu_fused/csv_analyzer.py | 6 +----- .../compute_advice/npu_fused/json_analyzer.py | 2 +- .../compute_advice/npu_fused/op_perf.py | 6 +++--- .../compute_advice/npu_fused_advice.py | 7 +++---- .../compute_advice/npu_slow_advice.py | 9 ++++----- .../advisor/advisor_backend/interface.py | 19 ++++++------------ .../overall_advice/overall_summary_advice.py | 6 ++---- .../timeline_advice/op_schedule_advice.py | 4 ++-- .../timeline_advice/optimizer_advice.py | 2 +- .../timeline_advice/timeline_advice_base.py | 4 +--- .../advisor/analyzer/analyzer_controller.py | 3 --- .../analyzer/cluster/slow_link_analyzer.py | 1 - .../pp_stage_computation_analyzer.py | 1 - .../graph_fusion/graph_fusion_checker.py | 1 - .../advisor/analyzer/memory/memory_checker.py | 2 -- .../fusion_ops/fusion_ops_analyzer.py | 2 -- .../analyzer/schedule/gc/gc_checker.py | 1 - .../schedule/timeline_base_checker.py | 3 +-- .../ai_core_freq/ai_core_freq_dataset.py | 5 ----- .../dataset/profiling/profiling_dataset.py | 4 ---- .../advisor/dataset/timeline_event_dataset.py | 1 - .../advisor/display/html/render.py | 1 - .../advisor/interface/interface.py | 6 ------ .../msprof_analyze/advisor/result/result.py | 3 +-- .../msprof_analyze/advisor/utils/utils.py | 3 --- profiler/msprof_analyze/cli/analyze_cli.py | 5 ----- profiler/msprof_analyze/cli/cluster_cli.py | 4 ---- profiler/msprof_analyze/cli/compare_cli.py | 4 ---- .../compare_tools/performance_compare.py | 8 +------- ...347\275\221URL\350\257\264\346\230\216.md" | 20 +++++++++---------- 43 files changed, 74 insertions(+), 147 deletions(-) diff --git a/README.md b/README.md index 064720548f..4c706cc6fe 100644 --- a/README.md +++ b/README.md @@ -30,21 +30,21 @@ [MindStudio Probe(msprobe,MindStudio 精度调试工具)](./debug/accuracy_tools/msprobe)。 -## [性能工具](./profiler) +## [性能工具](./profiler/msprof_analyze) -1. [compare_tools(性能比对工具)](./profiler/compare_tools) +1. [compare_tools(性能比对工具)](./profiler/msprof_analyze/compare_tools) 提供 NPU 与 GPU 性能拆解功能以及算子、通信、内存性能的比对功能。 -2. [cluster_analyse(集群分析工具)](./profiler/cluster_analyse) +2. [cluster_analyse(集群分析工具)](./profiler/msprof_analyze/cluster_analyse) 提供多机多卡的集群分析能力(基于通信域的通信分析和迭代耗时分析), 当前需要配合 MindStudio Insight 的集群分析功能使用。 -3. [advisor](./profiler/advisor) +3. [advisor](./profiler/msprof_analyze/advisor) 将 Ascend PyTorch Profiler 或者 msprof 采集的 PyTorch 场景性能数据进行分析,并输出性能调优建议。 -4. [bind_core](./profiler/affinity_cpu_bind) +4. [bind_core](./profiler/msprof_analyze/affinity_cpu_bind) 绑核脚本,支持非侵入修改工程代码,实现一键式绑核功能。 diff --git a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/cluster_advice_factory.py b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/cluster_advice_factory.py index 6bb93f4670..c3575c0a75 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/cluster_advice_factory.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/cluster_advice_factory.py @@ -12,12 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from advice_factory.advice_factory import AdviceFactory -from cluster_advice.slow_link_advice import SlowLinkAdvice -from cluster_advice.slow_rank_advice import SlowRankAdvice -from cluster_advice.cluster_pipeline_advice import ClusterPipelineAdvice -from cluster_advice.kernel_cluster_advice import KernelClusterAdvice -from common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.advice_factory.advice_factory import AdviceFactory +from msprof_analyze.advisor.advisor_backend.cluster_advice.slow_link_advice import SlowLinkAdvice +from msprof_analyze.advisor.advisor_backend.cluster_advice.slow_rank_advice import SlowRankAdvice +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_pipeline_advice import ClusterPipelineAdvice +from msprof_analyze.advisor.advisor_backend.cluster_advice.kernel_cluster_advice import KernelClusterAdvice +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant class ClusterAdviceFactory(AdviceFactory): diff --git a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/compute_advice_factory.py b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/compute_advice_factory.py index 336bef7dd8..026ea1675a 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/compute_advice_factory.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/compute_advice_factory.py @@ -12,10 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from common_func_advisor.constant import Constant -from advice_factory.advice_factory import AdviceFactory -from compute_advice.npu_fused_advice import NpuFusedAdvice -from compute_advice.npu_slow_advice import NpuSlowAdvice +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.advice_factory.advice_factory import AdviceFactory +from msprof_analyze.advisor.advisor_backend.compute_advice.npu_fused_advice import NpuFusedAdvice +from msprof_analyze.advisor.advisor_backend.compute_advice.npu_slow_advice import NpuSlowAdvice class ComputeAdviceFactory(AdviceFactory): diff --git a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/overall_advice_factory.py b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/overall_advice_factory.py index baf80cc200..9f4964aad4 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/overall_advice_factory.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/overall_advice_factory.py @@ -12,9 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from advice_factory.advice_factory import AdviceFactory -from common_func_advisor.constant import Constant -from overall_advice.overall_summary_advice import OverallSummaryAdvice +from msprof_analyze.advisor.advisor_backend.advice_factory.advice_factory import AdviceFactory +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.overall_advice.overall_summary_advice import OverallSummaryAdvice class OverallAdviceFactory(AdviceFactory): diff --git a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/timeline_advice_factory.py b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/timeline_advice_factory.py index 44b352e95a..2051c019b1 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/timeline_advice_factory.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/timeline_advice_factory.py @@ -12,10 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from advice_factory.advice_factory import AdviceFactory -from common_func_advisor.constant import Constant -from timeline_advice.optimizer_advice import OptimizerAdvice -from timeline_advice.op_schedule_advice import OpScheduleAdvice +from msprof_analyze.advisor.advisor_backend.advice_factory.advice_factory import AdviceFactory +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.timeline_advice.optimizer_advice import OptimizerAdvice +from msprof_analyze.advisor.advisor_backend.timeline_advice.op_schedule_advice import OpScheduleAdvice class TimelineAdviceFactory(AdviceFactory): diff --git a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_advice_base.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_advice_base.py index d5013fffb4..493947121c 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_advice_base.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_advice_base.py @@ -16,9 +16,8 @@ import os from abc import abstractmethod -from advice_base import AdviceBase -from cluster_analysis import Interface - +from msprof_analyze.advisor.advisor_backend.advice_base import AdviceBase +from cluster_analysis.interface import Interface from msprof_analyze.advisor.advisor_backend.logger import Logger from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py index 81fcf88ec0..25436cfb5b 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py @@ -26,12 +26,11 @@ from typing import List from typing import Optional from typing import Tuple -from cluster_advice.cluster_advice_base import ClusterAdviceBase +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_advice_base import ClusterAdviceBase from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor -from common_func_advisor.constant import Constant -from common_func_advisor.trace_view_preprocessor import FineTraceViewData -from common_func_advisor.trace_view_preprocessor import TraceViewPreProcessor - +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.common_func_advisor.trace_view_preprocessor import FineTraceViewData +from msprof_analyze.advisor.advisor_backend.common_func_advisor.trace_view_preprocessor import TraceViewPreProcessor from msprof_analyze.advisor.advisor_backend.logger import Logger from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py index e3b264463b..a7d3a01095 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py @@ -15,8 +15,8 @@ import os import pandas as pd -from common_func_advisor.constant import Constant as AdvisorConstant -from cluster_advice.cluster_advice_base import ClusterAdviceBase +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant as AdvisorConstant +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_advice_base import ClusterAdviceBase from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor from msprof_analyze.prof_common.file_manager import FileManager from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_link_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_link_advice.py index 564a844df3..2024adf8f6 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_link_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_link_advice.py @@ -15,8 +15,8 @@ import os from collections import defaultdict -from common_func_advisor.constant import Constant -from cluster_advice.cluster_advice_base import ClusterAdviceBase +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_advice_base import ClusterAdviceBase from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_rank_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_rank_advice.py index af644ed4b8..f6e0bf7dec 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_rank_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_rank_advice.py @@ -15,9 +15,9 @@ import os from collections import defaultdict -from common_func_advisor.constant import Constant -from cluster_advice.cluster_advice_base import ClusterAdviceBase -from prof_bean_advisor.cluster_step_trace_time_bean import ClusterStepTraceTimeBean +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_advice_base import ClusterAdviceBase +from msprof_analyze.advisor.advisor_backend.prof_bean_advisor.cluster_step_trace_time_bean import ClusterStepTraceTimeBean from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_json.py b/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_json.py index 74152d628b..5af97c785a 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_json.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_json.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - from abc import abstractmethod from dataclasses import dataclass from dataclasses import field diff --git a/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py b/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py index 028ee8d4b1..782cf459ed 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py @@ -19,7 +19,7 @@ import sys from typing import Optional from dataclasses import dataclass -from common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant from msprof_analyze.advisor.advisor_backend.logger import Logger logger = Logger() diff --git a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/compute_advice_base.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/compute_advice_base.py index 67e1513d81..a137b832e0 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/compute_advice_base.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/compute_advice_base.py @@ -18,7 +18,7 @@ from collections import defaultdict import os import logging -from advice_base import AdviceBase +from msprof_analyze.advisor.advisor_backend.advice_base import AdviceBase from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py index 1c7634a6e4..018c3c964d 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py @@ -13,14 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import multiprocessing - import pandas as pd -import numpy as np from common_func.path_manager import PathManager -from common_func_advisor.constant import Constant -from .op_perf import OpPerfFactory +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant class CSVAnalyzer: diff --git a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/json_analyzer.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/json_analyzer.py index 8a32e6d148..d47d6d8ad3 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/json_analyzer.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/json_analyzer.py @@ -17,7 +17,7 @@ import logging import pandas as pd -from common_func_advisor.trace_view_json import TraceViewJson +from msprof_analyze.advisor.advisor_backend.common_func_advisor.trace_view_json import TraceViewJson logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/op_perf.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/op_perf.py index 57488ae0f6..b8f5ef4285 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/op_perf.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/op_perf.py @@ -16,9 +16,9 @@ import functools from typing import Dict import logging -from common_func_advisor.constant import Constant -from common_func_advisor.constant import CoreType -from common_func_advisor.constant import PerfColor +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import CoreType +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import PerfColor logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused_advice.py index a8a7442264..349ede9905 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused_advice.py @@ -13,15 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import logging from abc import ABC import pandas as pd -from compute_advice.compute_advice_base import ComputeAdviceBase -from compute_advice.npu_fused.csv_analyzer import CSVAnalyzer -from compute_advice.npu_fused.json_analyzer import JSONAnalyzer +from msprof_analyze.advisor.advisor_backend.compute_advice.compute_advice_base import ComputeAdviceBase +from msprof_analyze.advisor.advisor_backend.compute_advice.npu_fused.csv_analyzer import CSVAnalyzer +from msprof_analyze.advisor.advisor_backend.compute_advice.npu_fused.json_analyzer import JSONAnalyzer logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py index fc271e035e..1eff2f6e99 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py @@ -20,11 +20,10 @@ import logging import pandas as pd from common_func.path_manager import PathManager -from compute_advice.compute_advice_base import ComputeAdviceBase -from compute_advice.npu_fused.op_perf import OpPerfFactory -from common_func_advisor.constant import Constant -from common_func_advisor.constant import PerfColor -from advisor_backend.common_func_advisor.trace_view_json import TraceViewJson +from msprof_analyze.advisor.advisor_backend.compute_advice.compute_advice_base import ComputeAdviceBase +from msprof_analyze.advisor.advisor_backend.compute_advice.npu_fused.op_perf import OpPerfFactory +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant, PerfColor +from msprof_analyze.advisor.advisor_backend.common_func_advisor.trace_view_json import TraceViewJson class NpuSlowAdvice(ComputeAdviceBase, ABC): diff --git a/profiler/msprof_analyze/advisor/advisor_backend/interface.py b/profiler/msprof_analyze/advisor/advisor_backend/interface.py index deb68822ec..f4e84f64e5 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/interface.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/interface.py @@ -13,19 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -import sys - -sys.path.append( - os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "advisor_backend")) -sys.path.append( - os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "compare_tools")) -sys.path.append( - os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "cluster_analyse")) -from common_func_advisor.constant import Constant -from advisor_backend.advice_factory.cluster_advice_factory import ClusterAdviceFactory -from advisor_backend.advice_factory.compute_advice_factory import ComputeAdviceFactory -from advisor_backend.advice_factory.timeline_advice_factory import TimelineAdviceFactory -from advisor_backend.advice_factory.overall_advice_factory import OverallAdviceFactory + +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.advice_factory.cluster_advice_factory import ClusterAdviceFactory +from msprof_analyze.advisor.advisor_backend.advice_factory.compute_advice_factory import ComputeAdviceFactory +from msprof_analyze.advisor.advisor_backend.advice_factory.timeline_advice_factory import TimelineAdviceFactory +from msprof_analyze.advisor.advisor_backend.advice_factory.overall_advice_factory import OverallAdviceFactory class Interface: diff --git a/profiler/msprof_analyze/advisor/advisor_backend/overall_advice/overall_summary_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/overall_advice/overall_summary_advice.py index 407f636335..979fcf7246 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/overall_advice/overall_summary_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/overall_advice/overall_summary_advice.py @@ -15,11 +15,9 @@ import os import logging -from compare_interface.comparison_interface import ComparisonInterface -from advisor_backend.advice_base import AdviceBase - +from msprof_analyze.compare_tools.compare_interface.comparison_interface import ComparisonInterface +from msprof_analyze.advisor.advisor_backend.advice_base import AdviceBase from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt -from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager from msprof_analyze.prof_common.constant import Constant logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/op_schedule_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/op_schedule_advice.py index 11008b2f37..ac163a7948 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/op_schedule_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/op_schedule_advice.py @@ -15,8 +15,8 @@ import logging from decimal import Decimal -from common_func_advisor.constant import Constant -from timeline_advice.timeline_advice_base import TimelineAdviceBase +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.timeline_advice.timeline_advice_base import TimelineAdviceBase logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/optimizer_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/optimizer_advice.py index 6d6bf7e9b9..6a01b1694e 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/optimizer_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/optimizer_advice.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from timeline_advice.timeline_advice_base import TimelineAdviceBase +from msprof_analyze.advisor.advisor_backend.timeline_advice.timeline_advice_base import TimelineAdviceBase class OptimizerAdvice(TimelineAdviceBase): diff --git a/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/timeline_advice_base.py b/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/timeline_advice_base.py index e712799a96..36c5e1e2de 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/timeline_advice_base.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/timeline_advice_base.py @@ -15,12 +15,10 @@ import logging import os -import json from abc import abstractmethod from collections import defaultdict -from advice_base import AdviceBase - +from msprof_analyze.advisor.advisor_backend.advice_base import AdviceBase from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/analyzer/analyzer_controller.py b/profiler/msprof_analyze/advisor/analyzer/analyzer_controller.py index a80966927d..6581d58d9f 100644 --- a/profiler/msprof_analyze/advisor/analyzer/analyzer_controller.py +++ b/profiler/msprof_analyze/advisor/analyzer/analyzer_controller.py @@ -24,9 +24,6 @@ from pathlib import Path import psutil -sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "compare_tools")) -sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "cluster_analyse")) - from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager from msprof_analyze.advisor.analyzer.cluster.slow_rank_analyzer import SlowRankAnalyzer from msprof_analyze.advisor.analyzer.cluster.slow_link_analyzer import SlowLinkAnalyzer diff --git a/profiler/msprof_analyze/advisor/analyzer/cluster/slow_link_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/cluster/slow_link_analyzer.py index 062b6548e1..9c4416009e 100644 --- a/profiler/msprof_analyze/advisor/analyzer/cluster/slow_link_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/cluster/slow_link_analyzer.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import defaultdict from typing import Dict, List import logging diff --git a/profiler/msprof_analyze/advisor/analyzer/computation/pp_stage_computation_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/computation/pp_stage_computation_analyzer.py index 538562cb1f..2780204b20 100644 --- a/profiler/msprof_analyze/advisor/analyzer/computation/pp_stage_computation_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/computation/pp_stage_computation_analyzer.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -import os from multiprocessing import Manager from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer diff --git a/profiler/msprof_analyze/advisor/analyzer/graph_fusion/graph_fusion_checker.py b/profiler/msprof_analyze/advisor/analyzer/graph_fusion/graph_fusion_checker.py index 1e2a66695d..727c5a3770 100644 --- a/profiler/msprof_analyze/advisor/analyzer/graph_fusion/graph_fusion_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/graph_fusion/graph_fusion_checker.py @@ -24,7 +24,6 @@ from msprof_analyze.advisor.common.graph.graph import Graph from msprof_analyze.advisor.common.graph.graph_parser import QueryGraphParser from msprof_analyze.advisor.dataset.graph_dataset import GraphDataset from msprof_analyze.advisor.common.graph.graph_match import find_isomorphisms -from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/analyzer/memory/memory_checker.py b/profiler/msprof_analyze/advisor/analyzer/memory/memory_checker.py index 7288bf8a5e..82bca84cd2 100644 --- a/profiler/msprof_analyze/advisor/analyzer/memory/memory_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/memory/memory_checker.py @@ -12,10 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os import re import logging -import yaml from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset, MemCollector from msprof_analyze.advisor.result.result import OptimizeResult diff --git a/profiler/msprof_analyze/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py index 1c5c325be1..247088080b 100644 --- a/profiler/msprof_analyze/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py @@ -22,7 +22,6 @@ from tqdm import tqdm from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt from msprof_analyze.prof_common.constant import Constant -from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes from msprof_analyze.advisor.common.timeline.event import TimelineEvent from msprof_analyze.advisor.config.config import Config from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset @@ -30,7 +29,6 @@ from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord from msprof_analyze.advisor.utils.utils import format_timeline_result from msprof_analyze.advisor.common.timeline.fusion_ops_db import init_timeline_ops_db from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor -from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/analyzer/schedule/gc/gc_checker.py b/profiler/msprof_analyze/advisor/analyzer/schedule/gc/gc_checker.py index 92a0d208f8..8e9f9139af 100644 --- a/profiler/msprof_analyze/advisor/analyzer/schedule/gc/gc_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/gc/gc_checker.py @@ -21,7 +21,6 @@ from msprof_analyze.advisor.result.result import OptimizeResult from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord from msprof_analyze.advisor.utils.utils import convert_to_float, convert_to_int from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager -from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/analyzer/schedule/timeline_base_checker.py b/profiler/msprof_analyze/advisor/analyzer/schedule/timeline_base_checker.py index 99ca99e99e..649e41e5da 100644 --- a/profiler/msprof_analyze/advisor/analyzer/schedule/timeline_base_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/timeline_base_checker.py @@ -12,14 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from abc import ABC, abstractmethod +from abc import ABC import multiprocessing import logging from msprof_analyze.prof_common.constant import Constant from msprof_analyze.advisor.common.timeline.event import TimelineEvent from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset -from msprof_analyze.advisor.result.result import OptimizeResult logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py b/profiler/msprof_analyze/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py index a7490c2e51..71b3e74107 100644 --- a/profiler/msprof_analyze/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py +++ b/profiler/msprof_analyze/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py @@ -16,11 +16,6 @@ import json import logging import math -import os -import traceback - -import ijson -from tqdm import tqdm from msprof_analyze.advisor.common.timeline.event import TimelineEvent from msprof_analyze.advisor.utils.utils import get_file_path_from_directory diff --git a/profiler/msprof_analyze/advisor/dataset/profiling/profiling_dataset.py b/profiler/msprof_analyze/advisor/dataset/profiling/profiling_dataset.py index 7981e4140f..65f70d69c6 100644 --- a/profiler/msprof_analyze/advisor/dataset/profiling/profiling_dataset.py +++ b/profiler/msprof_analyze/advisor/dataset/profiling/profiling_dataset.py @@ -18,10 +18,6 @@ import os import yaml from msprof_analyze.prof_common.constant import Constant -from msprof_analyze.advisor.common.profiling.ge_info import GeInfo -from msprof_analyze.advisor.common.profiling.msprof import Msprof -from msprof_analyze.advisor.common.profiling.op_summary import OpSummary -from msprof_analyze.advisor.common.profiling.tasktime import TaskTime from msprof_analyze.advisor.common.enum_params_parser import EnumParamsParser from msprof_analyze.advisor.dataset.dataset import Dataset from msprof_analyze.advisor.dataset.profiling.device_info import DeviceInfoParser diff --git a/profiler/msprof_analyze/advisor/dataset/timeline_event_dataset.py b/profiler/msprof_analyze/advisor/dataset/timeline_event_dataset.py index b9631445b8..c3ae3296f7 100644 --- a/profiler/msprof_analyze/advisor/dataset/timeline_event_dataset.py +++ b/profiler/msprof_analyze/advisor/dataset/timeline_event_dataset.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import inspect import logging import traceback from collections import OrderedDict diff --git a/profiler/msprof_analyze/advisor/display/html/render.py b/profiler/msprof_analyze/advisor/display/html/render.py index ffc07509ad..0921122067 100644 --- a/profiler/msprof_analyze/advisor/display/html/render.py +++ b/profiler/msprof_analyze/advisor/display/html/render.py @@ -20,7 +20,6 @@ from collections import defaultdict, OrderedDict from jinja2 import Environment, FileSystemLoader from msprof_analyze.prof_common.constant import Constant - from msprof_analyze.advisor.config.config import Config from msprof_analyze.advisor.utils.utils import singleton, safe_write diff --git a/profiler/msprof_analyze/advisor/interface/interface.py b/profiler/msprof_analyze/advisor/interface/interface.py index 4f9b6cba55..541213d42f 100644 --- a/profiler/msprof_analyze/advisor/interface/interface.py +++ b/profiler/msprof_analyze/advisor/interface/interface.py @@ -14,14 +14,8 @@ # limitations under the License. import os from collections import OrderedDict -import sys import logging -sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), - "cluster_analyse")) -sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), - "compare_tools")) - from msprof_analyze.advisor.utils.utils import Timer from msprof_analyze.advisor.result.result import OptimizeResult from msprof_analyze.advisor.analyzer.computation.profiling_analyzer import AicpuAnalyzer, BlockDimAnalyzer, \ diff --git a/profiler/msprof_analyze/advisor/result/result.py b/profiler/msprof_analyze/advisor/result/result.py index 2953ca5dbe..422fd43ec4 100644 --- a/profiler/msprof_analyze/advisor/result/result.py +++ b/profiler/msprof_analyze/advisor/result/result.py @@ -15,7 +15,6 @@ import json import os -import stat from collections import OrderedDict import click @@ -26,7 +25,7 @@ from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsMan from msprof_analyze.prof_common.constant import Constant from msprof_analyze.advisor.utils.utils import singleton, logger from msprof_analyze.advisor.config.config import Config -from msprof_analyze.advisor.utils.file import FdOpen, check_dir_writable +from msprof_analyze.advisor.utils.file import check_dir_writable from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/advisor/utils/utils.py b/profiler/msprof_analyze/advisor/utils/utils.py index 4f39da29ff..59c7701088 100644 --- a/profiler/msprof_analyze/advisor/utils/utils.py +++ b/profiler/msprof_analyze/advisor/utils/utils.py @@ -29,12 +29,9 @@ from functools import wraps from typing import Any, Set import ijson import click -import requests -from requests.adapters import HTTPAdapter from tqdm import tqdm from msprof_analyze.prof_common.constant import Constant -from msprof_analyze.advisor.common.version_control import VersionControl from msprof_analyze.advisor.utils.log import init_logger, get_log_level logger = logging.getLogger() diff --git a/profiler/msprof_analyze/cli/analyze_cli.py b/profiler/msprof_analyze/cli/analyze_cli.py index 73cbf6b749..69c77e04e0 100644 --- a/profiler/msprof_analyze/cli/analyze_cli.py +++ b/profiler/msprof_analyze/cli/analyze_cli.py @@ -13,14 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys -import os import logging import click -sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__)), "compare_tools")) -sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__)), "cluster_analyse")) - from msprof_analyze.prof_common.path_manager import PathManager from msprof_analyze.advisor.analyzer.analyzer_controller import AnalyzerController from msprof_analyze.advisor.utils.tools import CONTEXT_SETTINGS, ClickAliasedGroup diff --git a/profiler/msprof_analyze/cli/cluster_cli.py b/profiler/msprof_analyze/cli/cluster_cli.py index 1200642554..0cdb2bd2b1 100644 --- a/profiler/msprof_analyze/cli/cluster_cli.py +++ b/profiler/msprof_analyze/cli/cluster_cli.py @@ -12,12 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os -import sys import click -sys.path.append(os.path.dirname(os.path.dirname(__file__))) - from msprof_analyze.prof_common.constant import Constant from msprof_analyze.cluster_analyse.cluster_analysis import ALL_FEATURE_LIST, Interface from msprof_analyze.prof_common.path_manager import PathManager diff --git a/profiler/msprof_analyze/cli/compare_cli.py b/profiler/msprof_analyze/cli/compare_cli.py index c7af038d0c..7ef8139ae0 100644 --- a/profiler/msprof_analyze/cli/compare_cli.py +++ b/profiler/msprof_analyze/cli/compare_cli.py @@ -13,12 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import ast -import os -import sys import click -sys.path.append(os.path.dirname(os.path.dirname(__file__))) - from msprof_analyze.prof_common.path_manager import PathManager from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.analyze_dict import AnalyzeDict diff --git a/profiler/msprof_analyze/compare_tools/performance_compare.py b/profiler/msprof_analyze/compare_tools/performance_compare.py index a8fc94a8ee..478613e33c 100644 --- a/profiler/msprof_analyze/compare_tools/performance_compare.py +++ b/profiler/msprof_analyze/compare_tools/performance_compare.py @@ -15,14 +15,8 @@ import argparse import ast import datetime -import os.path -import sys -sys.path.append( - os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "cluster_analyse")) -sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) - -from compare_backend.comparison_generator import ComparisonGenerator +from msprof_analyze.compare_tools.compare_backend.comparison_generator import ComparisonGenerator from msprof_analyze.prof_common.analyze_dict import AnalyzeDict from msprof_analyze.prof_common.logger import get_logger from msprof_analyze.prof_common.path_manager import PathManager diff --git "a/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" "b/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" index 67064cd045..c78d206c1a 100644 --- "a/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" +++ "b/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" @@ -2,13 +2,13 @@ | 软件类型 | 软件名 | 路径 | 类型 | 内容 | 用途说明 | |------|----------------------------------------------------|------------------------------------------|------|------------------------------------------------------------------------------------------------------------|--------------------| -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md" | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0067.html | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/devtools/auxiliarydevtool/aoe_16_045.html | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://www.mindspore.cn/lite/docs/en/master/use/cloud_infer/converter_tool_ascend.html#aoe-auto-tuning | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0059.html | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/config/config.ini | 公网地址 | https://gitee.com/ascend/mstt/tree/master/profiler | msprof-analyze工具地址 | -| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/LICENSE | 公网地址 | http://www.apache.org/licenses/LICENSE-2.0 | 开源软件协议地址 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/rules/aicpu_rules.ymal | 公网地址 | https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20AI%20CPU%20Operator%20Replacement.md | AI CPU 算子替换样例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/rules/environment_variable_info.yaml | 公网地址 | https://support.huawei.com/enterprise/zh/doc/EDOC1100371278/5eeeed85?idPath=23710424 | 组网指南 | -| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/config/config.ini | 公网地址 | pmail_mindstudio@huawei.com | 公网邮箱 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/config/config.ini | 公网地址 | https://gitee.com/ascend/mstt/blob/master/profiler/msprof_analyze/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md" | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0067.html | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/devtools/auxiliarydevtool/aoe_16_045.html | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/config/config.ini | 公网地址 | https://www.mindspore.cn/lite/docs/en/master/use/cloud_infer/converter_tool_ascend.html#aoe-auto-tuning | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0059.html | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/msprof_analyze/config/config.ini | 公网地址 | https://gitee.com/ascend/mstt/tree/master/profiler/msprof_analyze | msprof-analyze工具地址 | +| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/msprof_analyze/LICENSE | 公网地址 | http://www.apache.org/licenses/LICENSE-2.0 | 开源软件协议地址 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/rules/aicpu_rules.ymal | 公网地址 | https://gitee.com/ascend/mstt/blob/master/profiler/msprof_analyze/advisor/doc/Samples%20of%20AI%20CPU%20Operator%20Replacement.md | AI CPU 算子替换样例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/rules/environment_variable_info.yaml | 公网地址 | https://support.huawei.com/enterprise/zh/doc/EDOC1100371278/5eeeed85?idPath=23710424 | 组网指南 | +| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/msprof_analyze/config/config.ini | 公网地址 | pmail_mindstudio@huawei.com | 公网邮箱 | -- Gitee From 0e359bfa61bf57136e290c324714349504b14712 Mon Sep 17 00:00:00 2001 From: hehongzhe <935062458@qq.com> Date: Wed, 5 Feb 2025 16:35:40 +0800 Subject: [PATCH 2/7] =?UTF-8?q?import=E6=95=B4=E6=94=B9=20=E5=89=A9?= =?UTF-8?q?=E4=BD=99=E9=83=A8=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../affinity_cpu_bind/README.md | 0 .../affinity_cpu_bind/bind_core.py | 0 .../{msprof_analyze => }/example/__init__.py | 0 .../example/mstx_torch_plugin/README.md | 0 .../example/mstx_torch_plugin/__init__.py | 0 .../example/mstx_torch_plugin/img/result.png | Bin .../mstx_torch_plugin/mstx_torch_plugin.py | 0 .../{msprof_analyze => }/example/setup.py | 0 .../advisor/cluster_perf_analysis.ipynb | 2 +- .../analysis/analysis_facade.py | 17 +++--- .../cluster_analyse/analysis/base_analysis.py | 3 +- .../analysis/cluster_base_info_analysis.py | 10 ++-- .../analysis/comm_matrix_analysis.py | 7 ++- .../analysis/communication_analysis.py | 12 ++--- .../analysis/host_info_analysis.py | 9 ++-- .../analysis/step_trace_time_analysis.py | 9 ++-- .../cluster_analyse/cluster_analysis.py | 8 +-- .../mindspore_data_preprocessor.py | 2 +- .../pytorch_data_preprocessor.py | 2 +- .../cluster_prof_Info_analysis.py | 4 -- .../cluster_utils/data_transfer_adapter.py | 2 +- .../parallel_strategy_calculator.py | 2 +- .../cluster_analyse/common_func/db_manager.py | 6 +-- .../base_communication_group.py | 2 +- .../communication_db_group.py | 4 +- .../communication_group_generator.py | 6 +-- .../communication_json_group.py | 2 +- .../prof_bean/communication_bandwidth_bean.py | 2 +- .../prof_bean/communication_time_bean.py | 2 +- .../recipes/cann_api_sum/cann_api_sum.py | 2 +- .../recipes/compute_op_sum/compute_op_sum.py | 2 +- .../recipes/hccl_sum/hccl_sum.py | 3 +- .../recipes/mstx_sum/mstx_sum.py | 2 +- .../comparator/api_compare_comparator.py | 5 +- .../comparator/communication_comparator.py | 6 +-- .../comparator/kernel_compare_comparator.py | 5 +- .../comparator/kernel_type_comparator.py | 7 ++- .../comparator/module_comparetor.py | 5 +- .../comparator/module_statistic_comparator.py | 4 +- .../comparator/operator_comparator.py | 2 +- .../operator_statistic_comparator.py | 4 +- .../comparator/overall_metrics_comparator.py | 4 +- .../overall_performance_comparator.py | 2 +- .../compare_bean/api_compare_bean.py | 4 +- .../compare_bean/communication_bean.py | 5 +- .../compare_bean/kernel_compare_bean.py | 5 +- .../compare_bean/kernel_type_compare_bean.py | 5 +- .../compare_bean/memory_compare_bean.py | 9 ++-- .../compare_bean/memory_statistic_bean.py | 7 ++- .../compare_bean/module_compare_bean.py | 11 ++-- .../compare_bean/module_statistic_bean.py | 5 +- .../compare_bean/operator_compare_bean.py | 9 ++-- .../compare_bean/operator_statistic_bean.py | 7 ++- .../origin_data_bean/compare_event.py | 3 +- .../origin_data_bean/kernel_details_bean.py | 5 +- .../origin_data_bean/memory_record_bean.py | 2 +- .../origin_data_bean/operator_memory_bean.py | 2 +- .../origin_data_bean/trace_event_bean.py | 5 +- .../compare_bean/overall_metrics_bean.py | 7 ++- .../compare_config/compare_config.py | 3 +- .../compare_backend/comparison_generator.py | 14 ++--- .../data_prepare/module_data_prepare.py | 9 ++-- .../data_prepare/operator_data_prepare.py | 5 +- .../data_prepare/sequence_pre_matching.py | 9 ++-- .../disaggregate/overall_perf_interface.py | 9 ++-- .../generator/detail_performance_generator.py | 49 +++++++++--------- .../overall_performance_generator.py | 9 ++-- .../interface/compare_interface.py | 21 ++++---- .../interface/overall_interface.py | 4 +- .../profiling_parser/base_profiling_parser.py | 8 +-- .../profiling_parser/gpu_profiling_parser.py | 4 +- .../profiling_parser/npu_profiling_parser.py | 11 ++-- .../compare_backend/utils/args_manager.py | 2 +- .../compare_backend/utils/module_node.py | 4 +- .../compare_backend/utils/name_function.py | 4 +- .../compare_backend/utils/torch_op_node.py | 5 +- .../compare_backend/utils/tree_builder.py | 6 +-- .../compare_backend/view/excel_view.py | 4 +- .../compare_backend/view/screen_view.py | 2 +- .../view/work_sheet_creator.py | 2 +- .../compare_interface/comparison_interface.py | 13 ++--- .../msprof_analyze/prof_common/__init__.py | 5 +- profiler/msprof_analyze/prof_common/utils.py | 4 +- ...st_advisor_cmd_single_ascend_pt_compare.py | 2 +- .../test_cluster_analyse_pytorch_db.py | 8 +-- .../test_cluster_advice_base.py | 2 +- .../test_cluster_pipeline_advice.py | 8 +-- .../test_kernel_cluster_advice.py | 2 +- .../cluster_advice/test_slow_link_advice.py | 2 +- .../cluster_advice/test_slow_rank_advice.py | 2 +- .../compute_advice/test_npu_slow_advice.py | 4 +- .../compute_advice/test_npufused_advice.py | 3 +- .../test_cluster_step_trace_time_bean.py | 7 +-- .../timeline_advice/test_opsche_advice.py | 6 +-- .../timeline_advice/test_optimizer_advice.py | 3 +- .../advisor/common/test_enum_params_parser.py | 6 --- .../test_bandwidth_contention_advice.py | 1 + .../test_byte_alignment_analyzer.py | 1 + .../test_packet_advice.py | 2 +- .../test_rdma_retransmission_advice.py | 2 +- .../compute_advice/test_frequency_advice.py | 2 +- .../test_conjectured_gc_advice.py | 1 + .../test_dataloader_checker.py | 1 - .../timeline_advice/test_gc_checker.py | 1 + .../timeline_advice/test_memory_op_checker.py | 1 - .../timeline_advice/test_syncbn_checker.py | 1 - .../test_synchronize_stream.py | 1 - .../test_timeline_op_collector.py | 3 -- .../test_pytorch_data_preprocessor.py | 2 +- .../test_step_trace_time_analysis.py | 7 +-- .../test_parallel_strategy_calculator.py | 2 +- .../common_func/test_file_manager.py | 2 +- .../test_communication_group_generator.py | 4 +- .../prof_bean/test_step_trace_time_bean.py | 2 +- .../recipes/test_compute_op_sum.py | 3 +- .../test_communication_comparator.py | 4 +- .../comparator/test_operator_comparator.py | 2 +- .../test_operator_statistic_comparator.py | 2 +- .../origin_data_bean/test_compare_event.py | 4 +- .../test_kernel_details_bean.py | 2 +- .../test_memory_record_bean.py | 2 +- .../test_operator_memory_bean.py | 2 +- .../origin_data_bean/test_trace_event_bean.py | 2 +- .../compare_bean/test_communication_bean.py | 2 +- .../compare_bean/test_memory_compare_bean.py | 2 +- .../test_memory_statistic_bean.py | 2 +- .../test_operator_compare_bean.py | 2 +- .../test_operator_statistic_bean.py | 2 +- .../compare_bean/test_profiling_info.py | 2 +- .../test_base_profiling_parser.py | 4 +- .../test_gpu_profiling_parser.py | 6 +-- .../test_npu_profiling_parser.py | 8 +-- .../compare_tools/utils/test_name_function.py | 6 +-- .../compare_tools/utils/test_tree_builder.py | 6 +-- .../ut/compare_tools/view/test_excel_view.py | 2 +- .../ut/compare_tools/view/test_screen_view.py | 2 +- .../view/test_worker_sheet_creator.py | 4 +- 137 files changed, 277 insertions(+), 350 deletions(-) rename profiler/{msprof_analyze => }/affinity_cpu_bind/README.md (100%) rename profiler/{msprof_analyze => }/affinity_cpu_bind/bind_core.py (100%) rename profiler/{msprof_analyze => }/example/__init__.py (100%) rename profiler/{msprof_analyze => }/example/mstx_torch_plugin/README.md (100%) rename profiler/{msprof_analyze => }/example/mstx_torch_plugin/__init__.py (100%) rename profiler/{msprof_analyze => }/example/mstx_torch_plugin/img/result.png (100%) rename profiler/{msprof_analyze => }/example/mstx_torch_plugin/mstx_torch_plugin.py (100%) rename profiler/{msprof_analyze => }/example/setup.py (100%) diff --git a/profiler/msprof_analyze/affinity_cpu_bind/README.md b/profiler/affinity_cpu_bind/README.md similarity index 100% rename from profiler/msprof_analyze/affinity_cpu_bind/README.md rename to profiler/affinity_cpu_bind/README.md diff --git a/profiler/msprof_analyze/affinity_cpu_bind/bind_core.py b/profiler/affinity_cpu_bind/bind_core.py similarity index 100% rename from profiler/msprof_analyze/affinity_cpu_bind/bind_core.py rename to profiler/affinity_cpu_bind/bind_core.py diff --git a/profiler/msprof_analyze/example/__init__.py b/profiler/example/__init__.py similarity index 100% rename from profiler/msprof_analyze/example/__init__.py rename to profiler/example/__init__.py diff --git a/profiler/msprof_analyze/example/mstx_torch_plugin/README.md b/profiler/example/mstx_torch_plugin/README.md similarity index 100% rename from profiler/msprof_analyze/example/mstx_torch_plugin/README.md rename to profiler/example/mstx_torch_plugin/README.md diff --git a/profiler/msprof_analyze/example/mstx_torch_plugin/__init__.py b/profiler/example/mstx_torch_plugin/__init__.py similarity index 100% rename from profiler/msprof_analyze/example/mstx_torch_plugin/__init__.py rename to profiler/example/mstx_torch_plugin/__init__.py diff --git a/profiler/msprof_analyze/example/mstx_torch_plugin/img/result.png b/profiler/example/mstx_torch_plugin/img/result.png similarity index 100% rename from profiler/msprof_analyze/example/mstx_torch_plugin/img/result.png rename to profiler/example/mstx_torch_plugin/img/result.png diff --git a/profiler/msprof_analyze/example/mstx_torch_plugin/mstx_torch_plugin.py b/profiler/example/mstx_torch_plugin/mstx_torch_plugin.py similarity index 100% rename from profiler/msprof_analyze/example/mstx_torch_plugin/mstx_torch_plugin.py rename to profiler/example/mstx_torch_plugin/mstx_torch_plugin.py diff --git a/profiler/msprof_analyze/example/setup.py b/profiler/example/setup.py similarity index 100% rename from profiler/msprof_analyze/example/setup.py rename to profiler/example/setup.py diff --git a/profiler/msprof_analyze/advisor/cluster_perf_analysis.ipynb b/profiler/msprof_analyze/advisor/cluster_perf_analysis.ipynb index e3b6b1afd7..0fc37222f6 100644 --- a/profiler/msprof_analyze/advisor/cluster_perf_analysis.ipynb +++ b/profiler/msprof_analyze/advisor/cluster_perf_analysis.ipynb @@ -675,7 +675,7 @@ "metadata": {}, "outputs": [], "source": [ - "from advisor_backend.interface import Interface\n", + "from msprof_analyze.advisor.advisor_backend.interface import Interface\n", "import matplotlib.pyplot as plt\n", "import numpy as np" ] diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/analysis_facade.py b/profiler/msprof_analyze/cluster_analyse/analysis/analysis_facade.py index 74c53e769f..aa9658f0c6 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/analysis_facade.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/analysis_facade.py @@ -12,18 +12,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import threading from multiprocessing import Process, Value, Lock from tqdm import tqdm -from analysis.communication_analysis import CommunicationAnalysis -from analysis.communication_analysis import CommunicationAnalysisOptimized -from analysis.comm_matrix_analysis import CommMatrixAnalysis -from analysis.comm_matrix_analysis import CommMatrixAnalysisOptimized -from analysis.step_trace_time_analysis import StepTraceTimeAnalysis -from analysis.host_info_analysis import HostInfoAnalysis -from analysis.cluster_base_info_analysis import ClusterBaseInfoAnalysis -from common_func.context import Context +from msprof_analyze.cluster_analyse.analysis.communication_analysis import CommunicationAnalysis +from msprof_analyze.cluster_analyse.analysis.communication_analysis import CommunicationAnalysisOptimized +from msprof_analyze.cluster_analyse.analysis.comm_matrix_analysis import CommMatrixAnalysis +from msprof_analyze.cluster_analyse.analysis.comm_matrix_analysis import CommMatrixAnalysisOptimized +from msprof_analyze.cluster_analyse.analysis.step_trace_time_analysis import StepTraceTimeAnalysis +from msprof_analyze.cluster_analyse.analysis.host_info_analysis import HostInfoAnalysis +from msprof_analyze.cluster_analyse.analysis.cluster_base_info_analysis import ClusterBaseInfoAnalysis +from msprof_analyze.cluster_analyse.common_func.context import Context from msprof_analyze.cluster_analyse.common_func.analysis_loader import get_class_from_name from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/base_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/base_analysis.py index 145cefa66c..59b824c1de 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/base_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/base_analysis.py @@ -14,8 +14,7 @@ # limitations under the License. from abc import abstractmethod -from cluster_utils.data_transfer_adapter import DataTransferAdapter - +from msprof_analyze.cluster_analyse.cluster_utils.data_transfer_adapter import DataTransferAdapter from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import FileManager from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/cluster_base_info_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/cluster_base_info_analysis.py index 00c3ebae08..4e2fad2067 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/cluster_base_info_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/cluster_base_info_analysis.py @@ -14,13 +14,11 @@ # limitations under the License. import json import os -import logging - -from analysis.base_analysis import BaseAnalysis -from common_func.db_manager import DBManager -from common_func.utils import increase_shared_value -from common_func.path_manager import PathManager +from msprof_analyze.cluster_analyse.analysis.base_analysis import BaseAnalysis +from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value +from msprof_analyze.cluster_analyse.common_func.path_manager import PathManager from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py index b3e3c8ddae..5ce5b1387a 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py @@ -16,10 +16,9 @@ import os from collections import defaultdict -from analysis.base_analysis import BaseAnalysis -from common_func.db_manager import DBManager -from common_func.utils import increase_shared_value - +from msprof_analyze.cluster_analyse.analysis.base_analysis import BaseAnalysis +from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py index 890507d2eb..be48fa90d2 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py @@ -16,12 +16,12 @@ import os from collections import defaultdict -from analysis.base_analysis import BaseAnalysis -from common_func.table_constant import TableConstant -from common_func.db_manager import DBManager -from common_func.utils import increase_shared_value -from prof_bean.communication_bandwidth_bean import CommunicationBandwidthBean -from prof_bean.communication_time_bean import CommunicationTimeBean +from msprof_analyze.cluster_analyse.analysis.base_analysis import BaseAnalysis +from msprof_analyze.cluster_analyse.common_func.table_constant import TableConstant +from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value +from msprof_analyze.cluster_analyse.prof_bean.communication_bandwidth_bean import CommunicationBandwidthBean +from msprof_analyze.cluster_analyse.prof_bean.communication_time_bean import CommunicationTimeBean from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py index abef72d5ff..dc43e313c5 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py @@ -15,11 +15,10 @@ import os -from analysis.base_analysis import BaseAnalysis -from common_func.db_manager import DBManager -from common_func.utils import increase_shared_value -from common_func.path_manager import PathManager - +from msprof_analyze.cluster_analyse.analysis.base_analysis import BaseAnalysis +from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value +from msprof_analyze.cluster_analyse.common_func.path_manager import PathManager from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py index 1fbc8b8dad..a7cffdd176 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py @@ -14,11 +14,10 @@ # limitations under the License. import os -from common_func.db_manager import DBManager -from common_func.utils import increase_shared_value -from cluster_utils.parallel_strategy_calculator import ParallelStrategyCalculator -from prof_bean.step_trace_time_bean import StepTraceTimeBean - +from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value +from msprof_analyze.cluster_analyse.cluster_utils.parallel_strategy_calculator import ParallelStrategyCalculator +from msprof_analyze.cluster_analyse.prof_bean.step_trace_time_bean import StepTraceTimeBean from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import FileManager from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/cluster_analysis.py b/profiler/msprof_analyze/cluster_analyse/cluster_analysis.py index 4cb486608f..6464bb732d 100644 --- a/profiler/msprof_analyze/cluster_analyse/cluster_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/cluster_analysis.py @@ -18,10 +18,10 @@ import sys sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) -from analysis.analysis_facade import AnalysisFacade -from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor -from cluster_data_preprocess.mindspore_data_preprocessor import MindsporeDataPreprocessor -from communication_group.communication_group_generator import CommunicationGroupGenerator +from msprof_analyze.cluster_analyse.analysis.analysis_facade import AnalysisFacade +from msprof_analyze.cluster_analyse.cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor +from msprof_analyze.cluster_analyse.cluster_data_preprocess.mindspore_data_preprocessor import MindsporeDataPreprocessor +from msprof_analyze.cluster_analyse.communication_group.communication_group_generator import CommunicationGroupGenerator from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/mindspore_data_preprocessor.py b/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/mindspore_data_preprocessor.py index 3a1a91230d..eaa14fb71f 100644 --- a/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/mindspore_data_preprocessor.py +++ b/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/mindspore_data_preprocessor.py @@ -14,7 +14,7 @@ # limitations under the License. from collections import defaultdict -from cluster_data_preprocess.data_preprocessor import DataPreprocessor +from msprof_analyze.cluster_analyse.cluster_data_preprocess.data_preprocessor import DataPreprocessor from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py b/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py index fb2be0860d..d270605906 100644 --- a/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py +++ b/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py @@ -15,7 +15,7 @@ from collections import defaultdict import os -from cluster_data_preprocess.data_preprocessor import DataPreprocessor +from msprof_analyze.cluster_analyse.cluster_data_preprocess.data_preprocessor import DataPreprocessor from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import FileManager from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py b/profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py index 78bf78ddfb..b1945bc98e 100644 --- a/profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py @@ -27,10 +27,6 @@ from plotly.subplots import make_subplots from plotly.offline import plot from msprof_analyze.prof_common.logger import get_logger - -sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - from msprof_analyze.prof_common.path_manager import PathManager from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager diff --git a/profiler/msprof_analyze/cluster_analyse/cluster_utils/data_transfer_adapter.py b/profiler/msprof_analyze/cluster_analyse/cluster_utils/data_transfer_adapter.py index a9ee57e57c..b45b5b3ffb 100644 --- a/profiler/msprof_analyze/cluster_analyse/cluster_utils/data_transfer_adapter.py +++ b/profiler/msprof_analyze/cluster_analyse/cluster_utils/data_transfer_adapter.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import copy -from common_func.table_constant import TableConstant +from msprof_analyze.cluster_analyse.common_func.table_constant import TableConstant from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/cluster_analyse/cluster_utils/parallel_strategy_calculator.py b/profiler/msprof_analyze/cluster_analyse/cluster_utils/parallel_strategy_calculator.py index 0f0a1809d9..d6a8dd4057 100644 --- a/profiler/msprof_analyze/cluster_analyse/cluster_utils/parallel_strategy_calculator.py +++ b/profiler/msprof_analyze/cluster_analyse/cluster_utils/parallel_strategy_calculator.py @@ -15,7 +15,7 @@ from enum import Enum from dataclasses import dataclass -from .parallel_algorithm import MegatronAlgorithm +from msprof_analyze.cluster_analyse.cluster_utils.parallel_algorithm import MegatronAlgorithm class ParallelAlgorithmType(Enum): diff --git a/profiler/msprof_analyze/cluster_analyse/common_func/db_manager.py b/profiler/msprof_analyze/cluster_analyse/common_func/db_manager.py index 7630413b56..b6045ba73a 100644 --- a/profiler/msprof_analyze/cluster_analyse/common_func/db_manager.py +++ b/profiler/msprof_analyze/cluster_analyse/common_func/db_manager.py @@ -16,9 +16,9 @@ import os import sqlite3 -from common_func.empty_class import EmptyClass -from common_func.tables_config import TablesConfig -from common_func.sql_extention_func import SqlExtentionAggregateFunc +from msprof_analyze.cluster_analyse.common_func.empty_class import EmptyClass +from msprof_analyze.cluster_analyse.common_func.tables_config import TablesConfig +from msprof_analyze.cluster_analyse.common_func.sql_extention_func import SqlExtentionAggregateFunc from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import check_db_path_valid diff --git a/profiler/msprof_analyze/cluster_analyse/communication_group/base_communication_group.py b/profiler/msprof_analyze/cluster_analyse/communication_group/base_communication_group.py index c15662ac0f..8f6625f8f6 100644 --- a/profiler/msprof_analyze/cluster_analyse/communication_group/base_communication_group.py +++ b/profiler/msprof_analyze/cluster_analyse/communication_group/base_communication_group.py @@ -19,7 +19,7 @@ from collections import defaultdict from copy import deepcopy from multiprocessing import Pool -from cluster_utils.data_transfer_adapter import DataTransferAdapter +from msprof_analyze.cluster_analyse.cluster_utils.data_transfer_adapter import DataTransferAdapter from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py index a3b1ef3725..55f0b7802c 100644 --- a/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py +++ b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py @@ -14,8 +14,8 @@ # limitations under the License. import os -from common_func.db_manager import DBManager -from communication_group.base_communication_group import BaseCommunicationGroup +from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.cluster_analyse.communication_group.base_communication_group import BaseCommunicationGroup from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/communication_group/communication_group_generator.py b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_group_generator.py index 00ced8e8bd..39a51e5070 100644 --- a/profiler/msprof_analyze/cluster_analyse/communication_group/communication_group_generator.py +++ b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_group_generator.py @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from communication_group.communication_db_group import CommunicationDBGroup -from communication_group.communication_db_group import CommunicationDBGroupOptimized -from communication_group.communication_json_group import CommunicationJsonGroup +from msprof_analyze.cluster_analyse.communication_group.communication_db_group import CommunicationDBGroup +from msprof_analyze.cluster_analyse.communication_group.communication_db_group import CommunicationDBGroupOptimized +from msprof_analyze.cluster_analyse.communication_group.communication_json_group import CommunicationJsonGroup from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/cluster_analyse/communication_group/communication_json_group.py b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_json_group.py index 2d39c8ef7d..9794822826 100644 --- a/profiler/msprof_analyze/cluster_analyse/communication_group/communication_json_group.py +++ b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_json_group.py @@ -15,7 +15,7 @@ import os -from communication_group.base_communication_group import BaseCommunicationGroup +from msprof_analyze.cluster_analyse.communication_group.base_communication_group import BaseCommunicationGroup from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_bandwidth_bean.py b/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_bandwidth_bean.py index 3c3190ea41..e3ae145541 100644 --- a/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_bandwidth_bean.py +++ b/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_bandwidth_bean.py @@ -14,7 +14,7 @@ # limitations under the License. -from common_func.table_constant import TableConstant +from msprof_analyze.cluster_analyse.common_func.table_constant import TableConstant class CommunicationBandwidthBean: diff --git a/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_time_bean.py b/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_time_bean.py index 5bc4603d29..704e9416aa 100644 --- a/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_time_bean.py +++ b/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_time_bean.py @@ -14,7 +14,7 @@ # limitations under the License. -from common_func.table_constant import TableConstant +from msprof_analyze.cluster_analyse.common_func.table_constant import TableConstant class CommunicationTimeBean: diff --git a/profiler/msprof_analyze/cluster_analyse/recipes/cann_api_sum/cann_api_sum.py b/profiler/msprof_analyze/cluster_analyse/recipes/cann_api_sum/cann_api_sum.py index 19ad6335d6..4fde5b7399 100644 --- a/profiler/msprof_analyze/cluster_analyse/recipes/cann_api_sum/cann_api_sum.py +++ b/profiler/msprof_analyze/cluster_analyse/recipes/cann_api_sum/cann_api_sum.py @@ -16,7 +16,7 @@ import os import pandas as pd -from common_func.utils import stdev +from msprof_analyze.cluster_analyse.common_func.utils import stdev from msprof_analyze.cluster_analyse.recipes.base_recipe_analysis import BaseRecipeAnalysis from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/recipes/compute_op_sum/compute_op_sum.py b/profiler/msprof_analyze/cluster_analyse/recipes/compute_op_sum/compute_op_sum.py index 2d6eaad75d..78b960a657 100644 --- a/profiler/msprof_analyze/cluster_analyse/recipes/compute_op_sum/compute_op_sum.py +++ b/profiler/msprof_analyze/cluster_analyse/recipes/compute_op_sum/compute_op_sum.py @@ -16,7 +16,7 @@ import os import pandas as pd -from common_func.utils import describe_duration +from msprof_analyze.cluster_analyse.common_func.utils import describe_duration from msprof_analyze.cluster_analyse.recipes.base_recipe_analysis import BaseRecipeAnalysis from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/recipes/hccl_sum/hccl_sum.py b/profiler/msprof_analyze/cluster_analyse/recipes/hccl_sum/hccl_sum.py index ff2bfac981..d9a34c52e1 100644 --- a/profiler/msprof_analyze/cluster_analyse/recipes/hccl_sum/hccl_sum.py +++ b/profiler/msprof_analyze/cluster_analyse/recipes/hccl_sum/hccl_sum.py @@ -16,8 +16,7 @@ import os import pandas as pd -from common_func.utils import describe_duration - +from msprof_analyze.cluster_analyse.common_func.utils import describe_duration from msprof_analyze.cluster_analyse.recipes.base_recipe_analysis import BaseRecipeAnalysis from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/recipes/mstx_sum/mstx_sum.py b/profiler/msprof_analyze/cluster_analyse/recipes/mstx_sum/mstx_sum.py index fbcc678767..e656c4abc9 100644 --- a/profiler/msprof_analyze/cluster_analyse/recipes/mstx_sum/mstx_sum.py +++ b/profiler/msprof_analyze/cluster_analyse/recipes/mstx_sum/mstx_sum.py @@ -17,7 +17,7 @@ from collections import namedtuple import os import pandas as pd -from common_func.utils import describe_duration +from msprof_analyze.cluster_analyse.common_func.utils import describe_duration from msprof_analyze.cluster_analyse.recipes.base_recipe_analysis import BaseRecipeAnalysis from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/api_compare_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/api_compare_comparator.py index 6dd4ed5fc7..bea4492557 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/api_compare_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/api_compare_comparator.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.utils.common_func import update_order_id - +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.utils.common_func import update_order_id from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/communication_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/communication_comparator.py index a9aa221d33..7710d2cbb7 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/communication_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/communication_comparator.py @@ -12,9 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.compare_bean.communication_bean import CommunicationBean -from compare_backend.utils.common_func import update_order_id +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.communication_bean import CommunicationBean +from msprof_analyze.compare_tools.compare_backend.utils.common_func import update_order_id from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_compare_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_compare_comparator.py index 51fce7c156..89e1d3e0ad 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_compare_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_compare_comparator.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.utils.common_func import update_order_id - +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.utils.common_func import update_order_id from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_type_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_type_comparator.py index ebca4346fa..44e117c82f 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_type_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_type_comparator.py @@ -12,10 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.compare_bean.origin_data_bean.op_stastic_bean import OpStatisticBean -from compare_backend.utils.common_func import update_order_id - +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.op_stastic_bean import OpStatisticBean +from msprof_analyze.compare_tools.compare_backend.utils.common_func import update_order_id from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_comparetor.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_comparetor.py index 3b13c24411..11c701a55a 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_comparetor.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_comparetor.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.utils.common_func import update_order_id - +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.utils.common_func import update_order_id from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_statistic_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_statistic_comparator.py index a31d1f32a5..7fee4a361d 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_statistic_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_statistic_comparator.py @@ -14,8 +14,8 @@ # limitations under the License. from collections import OrderedDict -from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.utils.common_func import update_order_id +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.utils.common_func import update_order_id class ModuleStatisticComparator(BaseComparator): diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_comparator.py index b6476d422e..1091da79c3 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_comparator.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator class OperatorComparator(BaseComparator): diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_statistic_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_statistic_comparator.py index 56610e9734..db1db36f72 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_statistic_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_statistic_comparator.py @@ -12,8 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.utils.common_func import update_order_id +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.utils.common_func import update_order_id class OperatorStatisticComparator(BaseComparator): diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_metrics_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_metrics_comparator.py index ad853f2db8..8a80262c6a 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_metrics_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_metrics_comparator.py @@ -14,8 +14,8 @@ # limitations under the License. from math import isclose -from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.utils.excel_config import ExcelConfig +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_performance_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_performance_comparator.py index 0eee314ee1..839ea14f1a 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_performance_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_performance_comparator.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/api_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/api_compare_bean.py index 5178485700..faf98fb96d 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/api_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/api_compare_bean.py @@ -12,8 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.excel_config import ExcelConfig +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/communication_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/communication_bean.py index f840eb653c..1801a75cb8 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/communication_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/communication_bean.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.excel_config import ExcelConfig -from compare_backend.utils.common_func import calculate_diff_ratio - +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py index ff10d94dd5..3f64f28ec6 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import calculate_diff_ratio, convert_to_float -from compare_backend.utils.excel_config import ExcelConfig - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio, convert_to_float +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_type_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_type_compare_bean.py index b129facc94..1434f42d5f 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_type_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_type_compare_bean.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.excel_config import ExcelConfig - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_compare_bean.py index 5e2720a2bb..7a269d6dbc 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_compare_bean.py @@ -12,11 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.excel_config import ExcelConfig -from compare_backend.utils.torch_op_node import TorchOpNode -from compare_backend.utils.tree_builder import TreeBuilder - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig +from msprof_analyze.compare_tools.compare_backend.utils.torch_op_node import TorchOpNode +from msprof_analyze.compare_tools.compare_backend.utils.tree_builder import TreeBuilder from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py index 18ca502e72..e4f7792b5e 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py @@ -12,10 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.tree_builder import TreeBuilder -from compare_backend.utils.excel_config import ExcelConfig - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.tree_builder import TreeBuilder +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_compare_bean.py index 9dee49acda..b87fc7409f 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_compare_bean.py @@ -12,12 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import longest_common_subsequence_matching, calculate_diff_ratio -from compare_backend.utils.excel_config import ExcelConfig -from compare_backend.utils.module_node import ModuleNode -from compare_backend.utils.name_function import NameFunction -from compare_backend.utils.torch_op_node import TorchOpNode - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import longest_common_subsequence_matching, calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig +from msprof_analyze.compare_tools.compare_backend.utils.module_node import ModuleNode +from msprof_analyze.compare_tools.compare_backend.utils.name_function import NameFunction +from msprof_analyze.compare_tools.compare_backend.utils.torch_op_node import TorchOpNode from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_statistic_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_statistic_bean.py index b4448d1eb9..2ca8b7e119 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_statistic_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_statistic_bean.py @@ -14,9 +14,8 @@ # limitations under the License. import re -from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.excel_config import ExcelConfig - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_compare_bean.py index b475a59ef5..3770afc844 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_compare_bean.py @@ -12,11 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.excel_config import ExcelConfig -from compare_backend.utils.torch_op_node import TorchOpNode -from compare_backend.utils.tree_builder import TreeBuilder - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig +from msprof_analyze.compare_tools.compare_backend.utils.torch_op_node import TorchOpNode +from msprof_analyze.compare_tools.compare_backend.utils.tree_builder import TreeBuilder from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py index 11d555576c..ab13584cb5 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py @@ -12,10 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.excel_config import ExcelConfig -from compare_backend.utils.tree_builder import TreeBuilder - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig +from msprof_analyze.compare_tools.compare_backend.utils.tree_builder import TreeBuilder from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py index 8c9d84075f..1f921334aa 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py @@ -14,8 +14,7 @@ # limitations under the License. from decimal import Decimal -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean - +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py index f1ff9f575a..d9f6e519da 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py @@ -17,9 +17,8 @@ from decimal import Decimal import pandas as pd -from compare_backend.utils.common_func import convert_to_float, convert_to_decimal -from compare_backend.compare_config.compare_config import CompareConfig - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import convert_to_float, convert_to_decimal +from msprof_analyze.compare_tools.compare_backend.compare_config.compare_config import CompareConfig from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/memory_record_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/memory_record_bean.py index 6d834d724d..e123d53fab 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/memory_record_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/memory_record_bean.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import convert_to_float +from msprof_analyze.compare_tools.compare_backend.utils.common_func import convert_to_float class MemoryRecordBean: diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/operator_memory_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/operator_memory_bean.py index 2d9d5c813f..71e5548fef 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/operator_memory_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/operator_memory_bean.py @@ -14,7 +14,7 @@ # limitations under the License. from decimal import Decimal -from compare_backend.utils.common_func import convert_to_float, convert_to_decimal +from msprof_analyze.compare_tools.compare_backend.utils.common_func import convert_to_float, convert_to_decimal class OperatorMemoryBean: diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py index 9afcab5ec0..ab12d640a1 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py @@ -14,9 +14,8 @@ # limitations under the License. from decimal import Decimal -from compare_backend.utils.common_func import convert_to_float, convert_to_decimal -from compare_backend.compare_config.compare_config import CompareConfig - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import convert_to_float, convert_to_decimal +from msprof_analyze.compare_tools.compare_backend.compare_config.compare_config import CompareConfig from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py index c6c8a03ba8..518718cb35 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py @@ -14,10 +14,9 @@ # limitations under the License. from math import isclose -from compare_backend.compare_bean.profiling_info import ProfilingInfo -from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.excel_config import ExcelConfig, CellFormatType - +from msprof_analyze.compare_tools.compare_backend.compare_bean.profiling_info import ProfilingInfo +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig, CellFormatType from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_config/compare_config.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_config/compare_config.py index a44c074ea6..4d044a1fa0 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_config/compare_config.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_config/compare_config.py @@ -14,8 +14,7 @@ # limitations under the License. import os -from compare_backend.utils.singleton import Singleton - +from msprof_analyze.compare_tools.compare_backend.utils.singleton import Singleton from msprof_analyze.prof_common.utils import SafeConfigReader diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparison_generator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparison_generator.py index 4a880822ae..2a8c97f8b4 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparison_generator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparison_generator.py @@ -12,13 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.generator.detail_performance_generator import DetailPerformanceGenerator -from compare_backend.generator.overall_performance_generator import OverallPerformanceGenerator -from compare_backend.interface.overall_interface import OverallInterface -from compare_backend.interface.compare_interface import CompareInterface -from compare_backend.profiling_parser.gpu_profiling_parser import GPUProfilingParser -from compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingParser -from compare_backend.utils.args_manager import ArgsManager +from msprof_analyze.compare_tools.compare_backend.generator.detail_performance_generator import DetailPerformanceGenerator +from msprof_analyze.compare_tools.compare_backend.generator.overall_performance_generator import OverallPerformanceGenerator +from msprof_analyze.compare_tools.compare_backend.interface.overall_interface import OverallInterface +from msprof_analyze.compare_tools.compare_backend.interface.compare_interface import CompareInterface +from msprof_analyze.compare_tools.compare_backend.profiling_parser.gpu_profiling_parser import GPUProfilingParser +from msprof_analyze.compare_tools.compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingParser +from msprof_analyze.compare_tools.compare_backend.utils.args_manager import ArgsManager from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/module_data_prepare.py b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/module_data_prepare.py index 52df18c78b..e8c56b2608 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/module_data_prepare.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/module_data_prepare.py @@ -15,11 +15,10 @@ import copy from queue import Queue -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.profiling_parser.base_profiling_parser import ProfilingResult -from compare_backend.utils.module_node import ModuleNode -from compare_backend.utils.tree_builder import TreeBuilder - +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import ProfilingResult +from msprof_analyze.compare_tools.compare_backend.utils.module_node import ModuleNode +from msprof_analyze.compare_tools.compare_backend.utils.tree_builder import TreeBuilder from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/operator_data_prepare.py b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/operator_data_prepare.py index 003a023221..b5da970126 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/operator_data_prepare.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/operator_data_prepare.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.profiling_parser.base_profiling_parser import ProfilingResult -from compare_backend.utils.tree_builder import TreeBuilder - +from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import ProfilingResult +from msprof_analyze.compare_tools.compare_backend.utils.tree_builder import TreeBuilder from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py index b8e674fe98..5c2590c723 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py @@ -14,11 +14,10 @@ # limitations under the License. from collections import deque -from compare_backend.utils.name_function import NameFunction -from compare_backend.utils.common_func import longest_common_subsequence_matching -from compare_backend.utils.torch_op_node import TorchOpNode -from compare_backend.utils.module_node import ModuleNode - +from msprof_analyze.compare_tools.compare_backend.utils.name_function import NameFunction +from msprof_analyze.compare_tools.compare_backend.utils.common_func import longest_common_subsequence_matching +from msprof_analyze.compare_tools.compare_backend.utils.torch_op_node import TorchOpNode +from msprof_analyze.compare_tools.compare_backend.utils.module_node import ModuleNode from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/disaggregate/overall_perf_interface.py b/profiler/msprof_analyze/compare_tools/compare_backend/disaggregate/overall_perf_interface.py index c59b35dd66..4dfbe55921 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/disaggregate/overall_perf_interface.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/disaggregate/overall_perf_interface.py @@ -12,11 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.profiling_parser.gpu_profiling_parser import GPUProfilingParser -from compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingParser -from compare_backend.utils.args_manager import ArgsManager -from compare_backend.utils.compare_args import Args - +from msprof_analyze.compare_tools.compare_backend.profiling_parser.gpu_profiling_parser import GPUProfilingParser +from msprof_analyze.compare_tools.compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingParser +from msprof_analyze.compare_tools.compare_backend.utils.args_manager import ArgsManager +from msprof_analyze.compare_tools.compare_backend.utils.compare_args import Args from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger from msprof_analyze.prof_common.path_manager import PathManager diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/generator/detail_performance_generator.py b/profiler/msprof_analyze/compare_tools/compare_backend/generator/detail_performance_generator.py index d48353be9c..d5d8518246 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/generator/detail_performance_generator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/generator/detail_performance_generator.py @@ -16,31 +16,30 @@ import os from collections import OrderedDict from datetime import datetime -from compare_backend.comparator.communication_comparator import CommunicationComparator -from compare_backend.comparator.module_comparetor import ModuleComparator -from compare_backend.comparator.module_statistic_comparator import ModuleStatisticComparator -from compare_backend.comparator.operator_comparator import OperatorComparator -from compare_backend.comparator.operator_statistic_comparator import OperatorStatisticComparator -from compare_backend.comparator.api_compare_comparator import ApiCompareComparator -from compare_backend.comparator.kernel_compare_comparator import KernelCompareComparator -from compare_backend.comparator.overall_metrics_comparator import OverallMetricsComparator -from compare_backend.compare_bean.communication_bean import CommunicationBean -from compare_backend.compare_bean.memory_compare_bean import MemoryCompareBean -from compare_backend.compare_bean.memory_statistic_bean import MemoryStatisticBean -from compare_backend.compare_bean.module_compare_bean import ModuleCompareBean -from compare_backend.compare_bean.module_statistic_bean import ModuleStatisticBean -from compare_backend.compare_bean.operator_compare_bean import OperatorCompareBean -from compare_backend.compare_bean.operator_statistic_bean import OperatorStatisticBean -from compare_backend.compare_bean.api_compare_bean import ApiCompareBean -from compare_backend.compare_bean.kernel_compare_bean import KernelCompareBean -from compare_backend.compare_bean.overall_metrics_bean import OverallMetricsBean -from compare_backend.data_prepare.module_data_prepare import ModuleDataPrepare -from compare_backend.data_prepare.operator_data_prepare import OperatorDataPrepare -from compare_backend.comparator.kernel_type_comparator import KernelTypeComparator -from compare_backend.compare_bean.kernel_type_compare_bean import KernelTypeCompareBean -from compare_backend.view.excel_view import ExcelView -from compare_backend.data_prepare.sequence_pre_matching import SequencePreMatching - +from msprof_analyze.compare_tools.compare_backend.comparator.communication_comparator import CommunicationComparator +from msprof_analyze.compare_tools.compare_backend.comparator.module_comparetor import ModuleComparator +from msprof_analyze.compare_tools.compare_backend.comparator.module_statistic_comparator import ModuleStatisticComparator +from msprof_analyze.compare_tools.compare_backend.comparator.operator_comparator import OperatorComparator +from msprof_analyze.compare_tools.compare_backend.comparator.operator_statistic_comparator import OperatorStatisticComparator +from msprof_analyze.compare_tools.compare_backend.comparator.api_compare_comparator import ApiCompareComparator +from msprof_analyze.compare_tools.compare_backend.comparator.kernel_compare_comparator import KernelCompareComparator +from msprof_analyze.compare_tools.compare_backend.comparator.overall_metrics_comparator import OverallMetricsComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.communication_bean import CommunicationBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.memory_compare_bean import MemoryCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.memory_statistic_bean import MemoryStatisticBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.module_compare_bean import ModuleCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.module_statistic_bean import ModuleStatisticBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.operator_compare_bean import OperatorCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.operator_statistic_bean import OperatorStatisticBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.api_compare_bean import ApiCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.kernel_compare_bean import KernelCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.overall_metrics_bean import OverallMetricsBean +from msprof_analyze.compare_tools.compare_backend.data_prepare.module_data_prepare import ModuleDataPrepare +from msprof_analyze.compare_tools.compare_backend.data_prepare.operator_data_prepare import OperatorDataPrepare +from msprof_analyze.compare_tools.compare_backend.comparator.kernel_type_comparator import KernelTypeComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.kernel_type_compare_bean import KernelTypeCompareBean +from msprof_analyze.compare_tools.compare_backend.view.excel_view import ExcelView +from msprof_analyze.compare_tools.compare_backend.data_prepare.sequence_pre_matching import SequencePreMatching from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/generator/overall_performance_generator.py b/profiler/msprof_analyze/compare_tools/compare_backend/generator/overall_performance_generator.py index a162353e89..d79cba7773 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/generator/overall_performance_generator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/generator/overall_performance_generator.py @@ -12,11 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.overall_performance_comparator import OverallPerformanceComparator -from compare_backend.compare_bean.profiling_info import ProfilingInfo -from compare_backend.generator.base_generator import BaseGenerator -from compare_backend.view.screen_view import ScreenView - +from msprof_analyze.compare_tools.compare_backend.comparator.overall_performance_comparator import OverallPerformanceComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.profiling_info import ProfilingInfo +from msprof_analyze.compare_tools.compare_backend.generator.base_generator import BaseGenerator +from msprof_analyze.compare_tools.compare_backend.view.screen_view import ScreenView from msprof_analyze.prof_common.logger import get_logger logger = get_logger() diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/interface/compare_interface.py b/profiler/msprof_analyze/compare_tools/compare_backend/interface/compare_interface.py index 58e4a0dbeb..2a459c7621 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/interface/compare_interface.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/interface/compare_interface.py @@ -12,17 +12,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.operator_comparator import OperatorComparator -from compare_backend.comparator.api_compare_comparator import ApiCompareComparator -from compare_backend.comparator.kernel_compare_comparator import KernelCompareComparator -from compare_backend.compare_bean.operator_compare_bean import OperatorCompareBean -from compare_backend.compare_bean.api_compare_bean import ApiCompareBean -from compare_backend.compare_bean.kernel_compare_bean import KernelCompareBean -from compare_backend.data_prepare.operator_data_prepare import OperatorDataPrepare -from compare_backend.data_prepare.sequence_pre_matching import SequencePreMatching -from compare_backend.comparator.kernel_type_comparator import KernelTypeComparator -from compare_backend.compare_bean.kernel_type_compare_bean import KernelTypeCompareBean - +from msprof_analyze.compare_tools.compare_backend.comparator.operator_comparator import OperatorComparator +from msprof_analyze.compare_tools.compare_backend.comparator.api_compare_comparator import ApiCompareComparator +from msprof_analyze.compare_tools.compare_backend.comparator.kernel_compare_comparator import KernelCompareComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.operator_compare_bean import OperatorCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.api_compare_bean import ApiCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.kernel_compare_bean import KernelCompareBean +from msprof_analyze.compare_tools.compare_backend.data_prepare.operator_data_prepare import OperatorDataPrepare +from msprof_analyze.compare_tools.compare_backend.data_prepare.sequence_pre_matching import SequencePreMatching +from msprof_analyze.compare_tools.compare_backend.comparator.kernel_type_comparator import KernelTypeComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.kernel_type_compare_bean import KernelTypeCompareBean from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/interface/overall_interface.py b/profiler/msprof_analyze/compare_tools/compare_backend/interface/overall_interface.py index ab1f2b23d0..e875ad9e81 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/interface/overall_interface.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/interface/overall_interface.py @@ -12,8 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.overall_performance_comparator import OverallPerformanceComparator -from compare_backend.compare_bean.profiling_info import ProfilingInfo +from msprof_analyze.compare_tools.compare_backend.comparator.overall_performance_comparator import OverallPerformanceComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.profiling_info import ProfilingInfo from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py index c9a31a9269..482e517e0b 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py @@ -17,10 +17,10 @@ from decimal import Decimal import ijson -from compare_backend.compare_bean.origin_data_bean.compare_event import KernelEvent, MemoryEvent -from compare_backend.compare_bean.origin_data_bean.kernel_details_bean import KernelDetailsBean -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.compare_bean.profiling_info import ProfilingInfo +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.compare_event import KernelEvent, MemoryEvent +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.kernel_details_bean import KernelDetailsBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.profiling_info import ProfilingInfo from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import FileManager from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index b69950deed..f5a6e564c5 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -15,8 +15,8 @@ import sys from collections import defaultdict, Counter -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.profiling_parser.base_profiling_parser import BaseProfilingParser +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import BaseProfilingParser from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 941acedf80..1d40832041 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -16,12 +16,11 @@ import os import sys from math import ceil -from compare_backend.compare_bean.origin_data_bean.kernel_details_bean import KernelDetailsBean -from compare_backend.compare_bean.origin_data_bean.memory_record_bean import MemoryRecordBean -from compare_backend.compare_bean.origin_data_bean.operator_memory_bean import OperatorMemoryBean -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.profiling_parser.base_profiling_parser import BaseProfilingParser - +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.kernel_details_bean import KernelDetailsBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.memory_record_bean import MemoryRecordBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.operator_memory_bean import OperatorMemoryBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import BaseProfilingParser from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.op_stastic_bean import OpStatisticBean from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/utils/args_manager.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/args_manager.py index 3e01918363..b682612922 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/utils/args_manager.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/utils/args_manager.py @@ -15,7 +15,7 @@ import os.path import re -from compare_backend.utils.singleton import Singleton +from msprof_analyze.compare_tools.compare_backend.utils.singleton import Singleton from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import FileManager from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/utils/module_node.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/module_node.py index 12e2f7a30d..98fa2ca129 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/utils/module_node.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/utils/module_node.py @@ -15,8 +15,8 @@ import re from math import ceil -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.utils.torch_op_node import TorchOpNode +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.utils.torch_op_node import TorchOpNode class ModuleNode: diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/utils/name_function.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/name_function.py index 8068f52f42..f1e2e90d22 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/utils/name_function.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/utils/name_function.py @@ -12,8 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.module_node import ModuleNode -from compare_backend.utils.torch_op_node import TorchOpNode +from msprof_analyze.compare_tools.compare_backend.utils.module_node import ModuleNode +from msprof_analyze.compare_tools.compare_backend.utils.torch_op_node import TorchOpNode class NameFunction: diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/utils/torch_op_node.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/torch_op_node.py index 9692241a96..2b72ad3d99 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/utils/torch_op_node.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/utils/torch_op_node.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.compare_bean.origin_data_bean.compare_event import MemoryEvent -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean - +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.compare_event import MemoryEvent +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/utils/tree_builder.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/tree_builder.py index 5a3e10aa42..6872bed55f 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/utils/tree_builder.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/utils/tree_builder.py @@ -15,9 +15,9 @@ from queue import Queue -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.utils.module_node import ModuleNode -from compare_backend.utils.torch_op_node import TorchOpNode +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.utils.module_node import ModuleNode +from msprof_analyze.compare_tools.compare_backend.utils.torch_op_node import TorchOpNode class TreeBuilder: diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/view/excel_view.py b/profiler/msprof_analyze/compare_tools/compare_backend/view/excel_view.py index 7cf3a8554c..6a094fdf3d 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/view/excel_view.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/view/excel_view.py @@ -16,8 +16,8 @@ import os from xlsxwriter import Workbook -from compare_backend.view.base_view import BaseView -from compare_backend.view.work_sheet_creator import WorkSheetCreator +from msprof_analyze.compare_tools.compare_backend.view.base_view import BaseView +from msprof_analyze.compare_tools.compare_backend.view.work_sheet_creator import WorkSheetCreator from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/view/screen_view.py b/profiler/msprof_analyze/compare_tools/compare_backend/view/screen_view.py index fb615979d2..5797d0bf8c 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/view/screen_view.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/view/screen_view.py @@ -14,7 +14,7 @@ # limitations under the License. from prettytable import PrettyTable -from compare_backend.view.base_view import BaseView +from msprof_analyze.compare_tools.compare_backend.view.base_view import BaseView class ScreenView(BaseView): diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/view/work_sheet_creator.py b/profiler/msprof_analyze/compare_tools/compare_backend/view/work_sheet_creator.py index 250a41cff3..b73f6df97e 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/view/work_sheet_creator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/view/work_sheet_creator.py @@ -14,7 +14,7 @@ # limitations under the License. from xlsxwriter import Workbook -from compare_backend.utils.excel_config import ExcelConfig, CellFormatType +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig, CellFormatType class WorkSheetCreator: diff --git a/profiler/msprof_analyze/compare_tools/compare_interface/comparison_interface.py b/profiler/msprof_analyze/compare_tools/compare_interface/comparison_interface.py index bb3a18eb4b..70da42d20e 100644 --- a/profiler/msprof_analyze/compare_tools/compare_interface/comparison_interface.py +++ b/profiler/msprof_analyze/compare_tools/compare_interface/comparison_interface.py @@ -12,16 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import sys -import os - -sys.path.append( - os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "cluster_analyse")) -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from compare_backend.comparison_generator import ComparisonGenerator -from compare_backend.disaggregate.overall_perf_interface import OverallPerfInterface -from compare_backend.utils.compare_args import Args +from msprof_analyze.compare_tools.compare_backend.comparison_generator import ComparisonGenerator +from msprof_analyze.compare_tools.compare_backend.disaggregate.overall_perf_interface import OverallPerfInterface +from msprof_analyze.compare_tools.compare_backend.utils.compare_args import Args from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.analyze_dict import AnalyzeDict from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/prof_common/__init__.py b/profiler/msprof_analyze/prof_common/__init__.py index 8b7e7544bb..de0604079e 100644 --- a/profiler/msprof_analyze/prof_common/__init__.py +++ b/profiler/msprof_analyze/prof_common/__init__.py @@ -11,7 +11,4 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. -import os -import sys -sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) \ No newline at end of file +# limitations under the License. \ No newline at end of file diff --git a/profiler/msprof_analyze/prof_common/utils.py b/profiler/msprof_analyze/prof_common/utils.py index f6a9f8618c..9cfb438770 100644 --- a/profiler/msprof_analyze/prof_common/utils.py +++ b/profiler/msprof_analyze/prof_common/utils.py @@ -18,8 +18,8 @@ from email.utils import parseaddr from typing import Dict, List from urllib.parse import urlparse -from .logger import get_logger -from .path_manager import PathManager +from msprof_analyze.prof_common.logger import get_logger +from msprof_analyze.prof_common.path_manager import PathManager logger = get_logger() diff --git a/profiler/msprof_analyze/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py b/profiler/msprof_analyze/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py index 3ec56be2f1..b6c10e97cb 100644 --- a/profiler/msprof_analyze/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py +++ b/profiler/msprof_analyze/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py @@ -8,7 +8,7 @@ import pandas as pd from bs4 import BeautifulSoup from msprof_analyze.prof_common.path_manager import PathManager -from .utils import get_files, execute_cmd +from msprof_analyze.prof_common.utils import get_files, execute_cmd class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): diff --git a/profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py b/profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py index ba9cd68062..2469e1bd27 100644 --- a/profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py +++ b/profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py @@ -22,10 +22,10 @@ import pandas as pd from msprof_analyze.test.st.utils import execute_cmd, select_count, select_by_query from msprof_analyze.prof_common.file_manager import FileManager from msprof_analyze.prof_common.path_manager import PathManager -from .cluster_communication_analyzer_bandwidth_db import ClusterCommunicationAnalyzerBandwidthDb -from .cluster_communication_analyzer_matrix_db import ClusterCommunicationAnalyzerMatrixDb -from .cluster_communication_analyzer_time_db import ClusterCommunicationAnalyzerTime -from .cluster_step_trace_time_db import ClusterStepTraceTimeDb +from msprof_analyze.test.st.cluster_analyse.cluster_communication_analyzer_bandwidth_db import ClusterCommunicationAnalyzerBandwidthDb +from msprof_analyze.test.st.cluster_analyse.cluster_communication_analyzer_matrix_db import ClusterCommunicationAnalyzerMatrixDb +from msprof_analyze.test.st.cluster_analyse.cluster_communication_analyzer_time_db import ClusterCommunicationAnalyzerTime +from msprof_analyze.test.st.cluster_analyse.cluster_step_trace_time_db import ClusterStepTraceTimeDb class TestClusterAnalysePytorchDb(TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_advice_base.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_advice_base.py index 6235c06efb..068e08f163 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_advice_base.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_advice_base.py @@ -4,7 +4,7 @@ import unittest from unittest import mock from unittest.mock import MagicMock -from advisor_backend.cluster_advice.cluster_advice_base import ClusterAdviceBase +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_advice_base import ClusterAdviceBase class MockChildClusterAdvice(ClusterAdviceBase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py index 9cb7a278fd..1c613935c2 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py @@ -3,10 +3,10 @@ from unittest import mock from collections import deque from collections import defaultdict -from advisor_backend.cluster_advice.cluster_pipeline_advice import ClusterPipelineAdvice -from advisor_backend.cluster_advice.cluster_pipeline_advice import FineTraceViewData -from advisor_backend.cluster_advice.cluster_pipeline_advice import PipelineTimeSlice -from advisor_backend.cluster_advice.cluster_pipeline_advice import PipelineTraceViewer +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_pipeline_advice import ClusterPipelineAdvice +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_pipeline_advice import FineTraceViewData +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_pipeline_advice import PipelineTimeSlice +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_pipeline_advice import PipelineTraceViewer class TestClusterPipelineAdvice(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py index 62dd532b22..c372924a4f 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py @@ -6,7 +6,7 @@ from unittest import mock from unittest.mock import MagicMock from msprof_analyze.prof_common.constant import Constant -from advisor_backend.cluster_advice.kernel_cluster_advice import KernelClusterAdvice +from msprof_analyze.advisor.advisor_backend.cluster_advice.kernel_cluster_advice import KernelClusterAdvice class TestClusterAdviceBase(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_link_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_link_advice.py index bf283cda5c..3749e4e00b 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_link_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_link_advice.py @@ -1,6 +1,6 @@ import unittest -from advisor_backend.cluster_advice.slow_link_advice import SlowLinkAdvice +from msprof_analyze.advisor.advisor_backend.cluster_advice.slow_link_advice import SlowLinkAdvice class TestSlowLinkAdvice(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_rank_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_rank_advice.py index 6a45553e1e..b4462aa4d4 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_rank_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_rank_advice.py @@ -1,6 +1,6 @@ import unittest -from advisor_backend.cluster_advice.slow_rank_advice import SlowRankAdvice +from msprof_analyze.advisor.advisor_backend.cluster_advice.slow_rank_advice import SlowRankAdvice class TestSlowRankAdvice(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npu_slow_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npu_slow_advice.py index 72e5dc1eeb..bf9ead4ca4 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npu_slow_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npu_slow_advice.py @@ -5,8 +5,8 @@ import stat import csv import unittest -from advisor_backend.interface import Interface -from advisor_backend.compute_advice.npu_slow_advice import NpuSlowAdvice +from msprof_analyze.advisor.advisor_backend.interface import Interface +from msprof_analyze.advisor.advisor_backend.compute_advice.npu_slow_advice import NpuSlowAdvice class TestNpuSlowAdvice(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npufused_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npufused_advice.py index bb7e34e1b3..755f0bf2a5 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npufused_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npufused_advice.py @@ -4,9 +4,8 @@ import shutil import stat import csv import unittest -import pytest -from advisor_backend.interface import Interface +from msprof_analyze.advisor.advisor_backend.interface import Interface class TestComputeAdvice(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py index 144561c417..17c9c29a04 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py @@ -1,11 +1,6 @@ -import os -import stat -import shutil import unittest -from unittest import mock -from unittest.mock import MagicMock -from advisor_backend.prof_bean_advisor.cluster_step_trace_time_bean import ClusterStepTraceTimeBean +from msprof_analyze.advisor.advisor_backend.prof_bean_advisor.cluster_step_trace_time_bean import ClusterStepTraceTimeBean class TestClusterStepTraceTimeBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_opsche_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_opsche_advice.py index 00024746cc..d6f17a167b 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_opsche_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_opsche_advice.py @@ -1,11 +1,7 @@ import os -import shutil -import stat -import json import unittest -import pytest -from advisor_backend.interface import Interface +from msprof_analyze.advisor.advisor_backend.interface import Interface class TestOpScheAdvice(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_optimizer_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_optimizer_advice.py index de9fbcb5ca..b47aeaf5dd 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_optimizer_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_optimizer_advice.py @@ -3,9 +3,8 @@ import shutil import stat import json import unittest -import pytest -from advisor_backend.interface import Interface +from msprof_analyze.advisor.advisor_backend.interface import Interface class TestOptimizerAdvice(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/common/test_enum_params_parser.py b/profiler/msprof_analyze/test/ut/advisor/common/test_enum_params_parser.py index 8f50972e46..2d6dbf0eeb 100644 --- a/profiler/msprof_analyze/test/ut/advisor/common/test_enum_params_parser.py +++ b/profiler/msprof_analyze/test/ut/advisor/common/test_enum_params_parser.py @@ -1,10 +1,4 @@ import unittest -import sys -import os - -profiler_root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) -sys.path.append(os.path.join(profiler_root_dir, "compare_tools")) -sys.path.append(os.path.join(profiler_root_dir, "cluster_analyse")) from msprof_analyze.advisor.common.enum_params_parser import EnumParamsParser from msprof_analyze.test.ut.advisor.advisor_backend.tools.tool import recover_env diff --git a/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py index 3e6bc74658..07d87b381d 100644 --- a/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py @@ -4,6 +4,7 @@ import stat import json import csv import unittest + from msprof_analyze.advisor.interface.interface import Interface from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes diff --git a/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_byte_alignment_analyzer.py b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_byte_alignment_analyzer.py index 8f8a084a85..3b9e55c81f 100644 --- a/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_byte_alignment_analyzer.py +++ b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_byte_alignment_analyzer.py @@ -3,6 +3,7 @@ import shutil import stat import json import unittest + from msprof_analyze.advisor.interface.interface import Interface from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes diff --git a/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_packet_advice.py b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_packet_advice.py index ef26392d8a..d517024c48 100644 --- a/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_packet_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_packet_advice.py @@ -2,8 +2,8 @@ import os import shutil import stat import json - import unittest + from msprof_analyze.advisor.interface.interface import Interface from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes diff --git a/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py index a0aa23f92d..93dafbc052 100644 --- a/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py @@ -2,8 +2,8 @@ import os import shutil import stat import json - import unittest + from msprof_analyze.advisor.interface.interface import Interface from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes diff --git a/profiler/msprof_analyze/test/ut/advisor/compute_advice/test_frequency_advice.py b/profiler/msprof_analyze/test/ut/advisor/compute_advice/test_frequency_advice.py index 9eb606e3d8..d7527cfb88 100644 --- a/profiler/msprof_analyze/test/ut/advisor/compute_advice/test_frequency_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/compute_advice/test_frequency_advice.py @@ -2,8 +2,8 @@ import os import shutil import stat import json - import unittest + from msprof_analyze.advisor.interface.interface import Interface from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes from msprof_analyze.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset diff --git a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_conjectured_gc_advice.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_conjectured_gc_advice.py index 07dc8a36b5..5dae952753 100644 --- a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_conjectured_gc_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_conjectured_gc_advice.py @@ -4,6 +4,7 @@ import stat import json import multiprocessing import unittest + from msprof_analyze.advisor.interface.interface import Interface from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes diff --git a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_dataloader_checker.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_dataloader_checker.py index 6737197d26..ac11201039 100644 --- a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_dataloader_checker.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_dataloader_checker.py @@ -1,6 +1,5 @@ import unittest import os -import sys import yaml from msprof_analyze.advisor.analyzer.dataloader.dataloader_checker import DataloaderChecker diff --git a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_gc_checker.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_gc_checker.py index 77f0814df3..35ad00c3c4 100644 --- a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_gc_checker.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_gc_checker.py @@ -4,6 +4,7 @@ import stat import json import multiprocessing import unittest + from msprof_analyze.advisor.interface.interface import Interface from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes diff --git a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_memory_op_checker.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_memory_op_checker.py index 8c14417771..04440ec43f 100644 --- a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_memory_op_checker.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_memory_op_checker.py @@ -1,6 +1,5 @@ import unittest import os -import sys import yaml from msprof_analyze.advisor.analyzer.memory.memory_checker import MemoryOpsChecker diff --git a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_syncbn_checker.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_syncbn_checker.py index aae3dfe1bb..28e9e5710e 100644 --- a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_syncbn_checker.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_syncbn_checker.py @@ -1,6 +1,5 @@ import unittest import os -import sys import yaml from msprof_analyze.advisor.analyzer.schedule.syncbn.syncbn_checker import SyncBNChecker diff --git a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_synchronize_stream.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_synchronize_stream.py index 6363fffb18..4e0bd7705b 100644 --- a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_synchronize_stream.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_synchronize_stream.py @@ -1,6 +1,5 @@ import unittest import os -import sys import yaml from msprof_analyze.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_checker import SynchronizeStreamChecker diff --git a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_timeline_op_collector.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_timeline_op_collector.py index 00bda93ad5..183f79cd15 100644 --- a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_timeline_op_collector.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_timeline_op_collector.py @@ -1,7 +1,4 @@ import unittest -import os -import sys -import yaml from msprof_analyze.advisor.dataset.timeline_op_collector.timeline_op_collector import ( OpCompileCollector, diff --git a/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_pytorch_data_preprocessor.py b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_pytorch_data_preprocessor.py index 6686c4a3ea..280b89a466 100644 --- a/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_pytorch_data_preprocessor.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_pytorch_data_preprocessor.py @@ -19,7 +19,7 @@ import shutil import unittest from unittest import mock -from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor +from msprof_analyze.cluster_analyse.cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor class TestPytorchDataPreprocessor(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_step_trace_time_analysis.py b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_step_trace_time_analysis.py index 9913faee51..b0142a67e7 100644 --- a/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_step_trace_time_analysis.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_step_trace_time_analysis.py @@ -13,13 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. - -import os import unittest -from unittest import mock -from analysis.step_trace_time_analysis import StepTraceTimeAnalysis -from prof_bean.step_trace_time_bean import StepTraceTimeBean +from msprof_analyze.cluster_analyse.analysis.step_trace_time_analysis import StepTraceTimeAnalysis +from msprof_analyze.cluster_analyse.prof_bean.step_trace_time_bean import StepTraceTimeBean from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_utils/test_parallel_strategy_calculator.py b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_utils/test_parallel_strategy_calculator.py index 2eb8b300ab..12cad26214 100644 --- a/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_utils/test_parallel_strategy_calculator.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_utils/test_parallel_strategy_calculator.py @@ -1,6 +1,6 @@ import unittest -from cluster_utils.parallel_strategy_calculator import ParallelStrategyCalculator +from msprof_analyze.cluster_analyse.cluster_utils.parallel_strategy_calculator import ParallelStrategyCalculator class TestParallelStrategyCalculator(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/cluster_analyse/common_func/test_file_manager.py b/profiler/msprof_analyze/test/ut/cluster_analyse/common_func/test_file_manager.py index 9dec7532fa..4b68eaf9b0 100644 --- a/profiler/msprof_analyze/test/ut/cluster_analyse/common_func/test_file_manager.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/common_func/test_file_manager.py @@ -6,7 +6,7 @@ import unittest import pytest from msprof_analyze.prof_common.file_manager import FileManager -from prof_bean.step_trace_time_bean import StepTraceTimeBean +from msprof_analyze.cluster_analyse.prof_bean.step_trace_time_bean import StepTraceTimeBean class TestFileManager(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py b/profiler/msprof_analyze/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py index a698a6a584..3ef480a58b 100644 --- a/profiler/msprof_analyze/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py @@ -13,12 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. - -import os import unittest from unittest import mock -from communication_group.communication_group_generator import CommunicationGroupGenerator +from msprof_analyze.cluster_analyse.communication_group.communication_group_generator import CommunicationGroupGenerator from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/test/ut/cluster_analyse/prof_bean/test_step_trace_time_bean.py b/profiler/msprof_analyze/test/ut/cluster_analyse/prof_bean/test_step_trace_time_bean.py index e369df4842..275b4d00eb 100644 --- a/profiler/msprof_analyze/test/ut/cluster_analyse/prof_bean/test_step_trace_time_bean.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/prof_bean/test_step_trace_time_bean.py @@ -1,6 +1,6 @@ import unittest -from prof_bean.step_trace_time_bean import StepTraceTimeBean +from msprof_analyze.cluster_analyse.prof_bean.step_trace_time_bean import StepTraceTimeBean class TestStepTraceTimeBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/cluster_analyse/recipes/test_compute_op_sum.py b/profiler/msprof_analyze/test/ut/cluster_analyse/recipes/test_compute_op_sum.py index 2fffa30444..314a213bce 100644 --- a/profiler/msprof_analyze/test/ut/cluster_analyse/recipes/test_compute_op_sum.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/recipes/test_compute_op_sum.py @@ -15,7 +15,8 @@ import unittest import pandas as pd -from recipes.compute_op_sum.compute_op_sum import ComputeOpSum + +from msprof_analyze.cluster_analyse.recipes.compute_op_sum.compute_op_sum import ComputeOpSum from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_communication_comparator.py b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_communication_comparator.py index 3cd1884c22..f44b7b3ddf 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_communication_comparator.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_communication_comparator.py @@ -1,7 +1,7 @@ import unittest -from compare_backend.comparator.communication_comparator import CommunicationComparator -from compare_backend.compare_bean.communication_bean import CommunicationBean +from msprof_analyze.compare_tools.compare_backend.comparator.communication_comparator import CommunicationComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.communication_bean import CommunicationBean class TestCommunicationComparator(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_comparator.py b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_comparator.py index cb51b5756c..cff56a146c 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_comparator.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_comparator.py @@ -1,6 +1,6 @@ import unittest -from compare_backend.comparator.operator_comparator import OperatorComparator +from msprof_analyze.compare_tools.compare_backend.comparator.operator_comparator import OperatorComparator class MockBean: diff --git a/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py index 133fa197f1..fe09c5ffab 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import patch -from compare_backend.comparator.operator_statistic_comparator import OperatorStatisticComparator +from msprof_analyze.compare_tools.compare_backend.comparator.operator_statistic_comparator import OperatorStatisticComparator class MockBean: diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_compare_event.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_compare_event.py index 771e199339..175292ec40 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_compare_event.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_compare_event.py @@ -1,7 +1,7 @@ import unittest -from compare_backend.compare_bean.origin_data_bean.compare_event import KernelEvent, MemoryEvent -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.compare_event import KernelEvent, MemoryEvent +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean class TestKernelEvent(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py index 869ee85570..30b46b903b 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py @@ -1,6 +1,6 @@ import unittest -from compare_backend.compare_bean.origin_data_bean.kernel_details_bean import KernelDetailsBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.kernel_details_bean import KernelDetailsBean class TestKernelDetailsBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_memory_record_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_memory_record_bean.py index 3ec34ffbaa..b32cb3299c 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_memory_record_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_memory_record_bean.py @@ -1,6 +1,6 @@ import unittest -from compare_backend.compare_bean.origin_data_bean.memory_record_bean import MemoryRecordBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.memory_record_bean import MemoryRecordBean class TestMemoryRecordBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_operator_memory_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_operator_memory_bean.py index 027b620e81..cb5cf450b1 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_operator_memory_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_operator_memory_bean.py @@ -1,6 +1,6 @@ import unittest -from compare_backend.compare_bean.origin_data_bean.operator_memory_bean import OperatorMemoryBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.operator_memory_bean import OperatorMemoryBean class TestOperatorMemoryBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_trace_event_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_trace_event_bean.py index 2bd56e611d..8dbb9ac5d4 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_trace_event_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_trace_event_bean.py @@ -1,6 +1,6 @@ import unittest -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean class TestTraceEventBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_communication_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_communication_bean.py index 2b5ee56828..4c14de0a43 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_communication_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_communication_bean.py @@ -1,6 +1,6 @@ import unittest -from compare_backend.compare_bean.communication_bean import CommunicationBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.communication_bean import CommunicationBean class TestCommunicationBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_compare_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_compare_bean.py index f275c2261a..5d8576872a 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_compare_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_compare_bean.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import patch -from compare_backend.compare_bean.memory_compare_bean import MemoryCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.memory_compare_bean import MemoryCompareBean class MockNode: diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_statistic_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_statistic_bean.py index 40bed2160b..ae55d07121 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_statistic_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_statistic_bean.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import patch -from compare_backend.compare_bean.memory_statistic_bean import MemoryStatisticBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.memory_statistic_bean import MemoryStatisticBean class MockMemory: diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_compare_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_compare_bean.py index ab8dec156f..cc58d61c6d 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_compare_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_compare_bean.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import patch -from compare_backend.compare_bean.operator_compare_bean import OperatorCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.operator_compare_bean import OperatorCompareBean class MockNode: diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_statistic_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_statistic_bean.py index 4c4e338ce5..9a4b70198e 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_statistic_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_statistic_bean.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import patch -from compare_backend.compare_bean.operator_statistic_bean import OperatorStatisticBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.operator_statistic_bean import OperatorStatisticBean class MockKernel: diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_profiling_info.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_profiling_info.py index 59525f18f9..b408f734a3 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_profiling_info.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_profiling_info.py @@ -1,6 +1,6 @@ import unittest -from compare_backend.compare_bean.profiling_info import ProfilingInfo +from msprof_analyze.compare_tools.compare_backend.compare_bean.profiling_info import ProfilingInfo class TestProfilingInfo(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py index c4c26bd6dd..853e6a5ecf 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py @@ -1,8 +1,8 @@ import unittest from unittest.mock import patch -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.profiling_parser.base_profiling_parser import BaseProfilingParser, ProfilingResult +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import BaseProfilingParser, ProfilingResult class ProfilingParser(BaseProfilingParser): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py index 59ca545e2d..767e078bcc 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py @@ -2,9 +2,9 @@ import unittest from collections import defaultdict from unittest.mock import patch -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.profiling_parser.base_profiling_parser import ProfilingResult -from compare_backend.profiling_parser.gpu_profiling_parser import GPUProfilingParser +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import ProfilingResult +from msprof_analyze.compare_tools.compare_backend.profiling_parser.gpu_profiling_parser import GPUProfilingParser class TestGpuProfilingParser(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py index a835dd9c12..69159aa2b9 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py @@ -1,10 +1,10 @@ import unittest from unittest.mock import patch -from compare_backend.compare_bean.origin_data_bean.operator_memory_bean import OperatorMemoryBean -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.profiling_parser.base_profiling_parser import ProfilingResult -from compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingParser +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.operator_memory_bean import OperatorMemoryBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import ProfilingResult +from msprof_analyze.compare_tools.compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingParser class TestNPUProfilingParser(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/utils/test_name_function.py b/profiler/msprof_analyze/test/ut/compare_tools/utils/test_name_function.py index 2903f9838b..99b90051a4 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/utils/test_name_function.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/utils/test_name_function.py @@ -1,8 +1,8 @@ import unittest -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.utils.name_function import NameFunction -from compare_backend.utils.torch_op_node import TorchOpNode +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.utils.name_function import NameFunction +from msprof_analyze.compare_tools.compare_backend.utils.torch_op_node import TorchOpNode class Args: diff --git a/profiler/msprof_analyze/test/ut/compare_tools/utils/test_tree_builder.py b/profiler/msprof_analyze/test/ut/compare_tools/utils/test_tree_builder.py index 326a424d3d..389d11c5f3 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/utils/test_tree_builder.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/utils/test_tree_builder.py @@ -1,8 +1,8 @@ import unittest -from compare_backend.compare_bean.origin_data_bean.compare_event import MemoryEvent -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.utils.tree_builder import TreeBuilder +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.compare_event import MemoryEvent +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.utils.tree_builder import TreeBuilder class TestUtils(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/view/test_excel_view.py b/profiler/msprof_analyze/test/ut/compare_tools/view/test_excel_view.py index aa500c4242..cabc23877c 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/view/test_excel_view.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/view/test_excel_view.py @@ -2,7 +2,7 @@ import os import unittest from unittest.mock import patch -from compare_backend.view.excel_view import ExcelView +from msprof_analyze.compare_tools.compare_backend.view.excel_view import ExcelView class TestExcelView(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/view/test_screen_view.py b/profiler/msprof_analyze/test/ut/compare_tools/view/test_screen_view.py index caa25e396e..027cebbb9c 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/view/test_screen_view.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/view/test_screen_view.py @@ -1,6 +1,6 @@ import unittest -from compare_backend.view.screen_view import ScreenView +from msprof_analyze.compare_tools.compare_backend.view.screen_view import ScreenView class TestScreenView(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/view/test_worker_sheet_creator.py b/profiler/msprof_analyze/test/ut/compare_tools/view/test_worker_sheet_creator.py index 1e80931ff5..587cccca98 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/view/test_worker_sheet_creator.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/view/test_worker_sheet_creator.py @@ -4,8 +4,8 @@ import unittest import pandas as pd from xlsxwriter import Workbook -from compare_backend.utils.excel_config import ExcelConfig -from compare_backend.view.work_sheet_creator import WorkSheetCreator +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig +from msprof_analyze.compare_tools.compare_backend.view.work_sheet_creator import WorkSheetCreator class TestWorkerSheetCreator(unittest.TestCase): -- Gitee From 8c9aaa163ed236bf54050858f30153979ed874a5 Mon Sep 17 00:00:00 2001 From: hehongzhe <935062458@qq.com> Date: Wed, 5 Feb 2025 16:57:45 +0800 Subject: [PATCH 3/7] cleancode --- .../advisor_backend/cluster_advice/slow_rank_advice.py | 3 ++- .../compare_backend/compare_bean/module_compare_bean.py | 5 ++++- .../compare_backend/comparison_generator.py | 6 ++++-- .../generator/detail_performance_generator.py | 6 ++++-- .../generator/overall_performance_generator.py | 3 ++- .../compare_backend/interface/overall_interface.py | 3 ++- .../profiling_parser/base_profiling_parser.py | 8 ++++++-- .../profiling_parser/npu_profiling_parser.py | 9 ++++++--- .../cluster_analyse/test_cluster_analyse_pytorch_db.py | 9 ++++++--- .../test_cluster_step_trace_time_bean.py | 3 ++- .../comparator/test_operator_statistic_comparator.py | 3 ++- .../compare_bean/origin_data_bean/test_compare_event.py | 5 ++++- .../origin_data_bean/test_kernel_details_bean.py | 3 ++- .../origin_data_bean/test_memory_record_bean.py | 3 ++- .../origin_data_bean/test_operator_memory_bean.py | 3 ++- .../profiling_parser/test_base_profiling_parser.py | 5 ++++- .../profiling_parser/test_npu_profiling_parser.py | 3 ++- 17 files changed, 56 insertions(+), 24 deletions(-) diff --git a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_rank_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_rank_advice.py index f6e0bf7dec..182249e3a0 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_rank_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_rank_advice.py @@ -17,7 +17,8 @@ import os from collections import defaultdict from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_advice_base import ClusterAdviceBase -from msprof_analyze.advisor.advisor_backend.prof_bean_advisor.cluster_step_trace_time_bean import ClusterStepTraceTimeBean +from msprof_analyze.advisor.advisor_backend.prof_bean_advisor.cluster_step_trace_time_bean \ + import ClusterStepTraceTimeBean from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_compare_bean.py index b87fc7409f..107a2d79ca 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_compare_bean.py @@ -12,7 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from msprof_analyze.compare_tools.compare_backend.utils.common_func import longest_common_subsequence_matching, calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.common_func import ( + longest_common_subsequence_matching, + calculate_diff_ratio +) from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig from msprof_analyze.compare_tools.compare_backend.utils.module_node import ModuleNode from msprof_analyze.compare_tools.compare_backend.utils.name_function import NameFunction diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparison_generator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparison_generator.py index 2a8c97f8b4..2c3c3d920c 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparison_generator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparison_generator.py @@ -12,8 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from msprof_analyze.compare_tools.compare_backend.generator.detail_performance_generator import DetailPerformanceGenerator -from msprof_analyze.compare_tools.compare_backend.generator.overall_performance_generator import OverallPerformanceGenerator +from msprof_analyze.compare_tools.compare_backend.generator.detail_performance_generator \ + import DetailPerformanceGenerator +from msprof_analyze.compare_tools.compare_backend.generator.overall_performance_generator \ + import OverallPerformanceGenerator from msprof_analyze.compare_tools.compare_backend.interface.overall_interface import OverallInterface from msprof_analyze.compare_tools.compare_backend.interface.compare_interface import CompareInterface from msprof_analyze.compare_tools.compare_backend.profiling_parser.gpu_profiling_parser import GPUProfilingParser diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/generator/detail_performance_generator.py b/profiler/msprof_analyze/compare_tools/compare_backend/generator/detail_performance_generator.py index d5d8518246..6479d624fa 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/generator/detail_performance_generator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/generator/detail_performance_generator.py @@ -18,9 +18,11 @@ from datetime import datetime from msprof_analyze.compare_tools.compare_backend.comparator.communication_comparator import CommunicationComparator from msprof_analyze.compare_tools.compare_backend.comparator.module_comparetor import ModuleComparator -from msprof_analyze.compare_tools.compare_backend.comparator.module_statistic_comparator import ModuleStatisticComparator +from msprof_analyze.compare_tools.compare_backend.comparator.module_statistic_comparator \ + import ModuleStatisticComparator from msprof_analyze.compare_tools.compare_backend.comparator.operator_comparator import OperatorComparator -from msprof_analyze.compare_tools.compare_backend.comparator.operator_statistic_comparator import OperatorStatisticComparator +from msprof_analyze.compare_tools.compare_backend.comparator.operator_statistic_comparator \ + import OperatorStatisticComparator from msprof_analyze.compare_tools.compare_backend.comparator.api_compare_comparator import ApiCompareComparator from msprof_analyze.compare_tools.compare_backend.comparator.kernel_compare_comparator import KernelCompareComparator from msprof_analyze.compare_tools.compare_backend.comparator.overall_metrics_comparator import OverallMetricsComparator diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/generator/overall_performance_generator.py b/profiler/msprof_analyze/compare_tools/compare_backend/generator/overall_performance_generator.py index d79cba7773..e144ef185c 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/generator/overall_performance_generator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/generator/overall_performance_generator.py @@ -12,7 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from msprof_analyze.compare_tools.compare_backend.comparator.overall_performance_comparator import OverallPerformanceComparator +from msprof_analyze.compare_tools.compare_backend.comparator.overall_performance_comparator \ + import OverallPerformanceComparator from msprof_analyze.compare_tools.compare_backend.compare_bean.profiling_info import ProfilingInfo from msprof_analyze.compare_tools.compare_backend.generator.base_generator import BaseGenerator from msprof_analyze.compare_tools.compare_backend.view.screen_view import ScreenView diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/interface/overall_interface.py b/profiler/msprof_analyze/compare_tools/compare_backend/interface/overall_interface.py index e875ad9e81..dcd7c2b49e 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/interface/overall_interface.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/interface/overall_interface.py @@ -12,7 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from msprof_analyze.compare_tools.compare_backend.comparator.overall_performance_comparator import OverallPerformanceComparator +from msprof_analyze.compare_tools.compare_backend.comparator.overall_performance_comparator \ + import OverallPerformanceComparator from msprof_analyze.compare_tools.compare_backend.compare_bean.profiling_info import ProfilingInfo from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py index 482e517e0b..b3d9a29944 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py @@ -17,8 +17,12 @@ from decimal import Decimal import ijson -from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.compare_event import KernelEvent, MemoryEvent -from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.kernel_details_bean import KernelDetailsBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.compare_event import ( + KernelEvent, + MemoryEvent +) +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.kernel_details_bean \ + import KernelDetailsBean from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean from msprof_analyze.compare_tools.compare_backend.compare_bean.profiling_info import ProfilingInfo from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 1d40832041..6e7e9e1c1e 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -16,9 +16,12 @@ import os import sys from math import ceil -from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.kernel_details_bean import KernelDetailsBean -from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.memory_record_bean import MemoryRecordBean -from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.operator_memory_bean import OperatorMemoryBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.kernel_details_bean \ + import KernelDetailsBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.memory_record_bean \ + import MemoryRecordBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.operator_memory_bean \ + import OperatorMemoryBean from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import BaseProfilingParser from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.op_stastic_bean import OpStatisticBean diff --git a/profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py b/profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py index 2469e1bd27..bbc07adebf 100644 --- a/profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py +++ b/profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py @@ -22,9 +22,12 @@ import pandas as pd from msprof_analyze.test.st.utils import execute_cmd, select_count, select_by_query from msprof_analyze.prof_common.file_manager import FileManager from msprof_analyze.prof_common.path_manager import PathManager -from msprof_analyze.test.st.cluster_analyse.cluster_communication_analyzer_bandwidth_db import ClusterCommunicationAnalyzerBandwidthDb -from msprof_analyze.test.st.cluster_analyse.cluster_communication_analyzer_matrix_db import ClusterCommunicationAnalyzerMatrixDb -from msprof_analyze.test.st.cluster_analyse.cluster_communication_analyzer_time_db import ClusterCommunicationAnalyzerTime +from msprof_analyze.test.st.cluster_analyse.cluster_communication_analyzer_bandwidth_db \ + import ClusterCommunicationAnalyzerBandwidthDb +from msprof_analyze.test.st.cluster_analyse.cluster_communication_analyzer_matrix_db \ + import ClusterCommunicationAnalyzerMatrixDb +from msprof_analyze.test.st.cluster_analyse.cluster_communication_analyzer_time_db \ + import ClusterCommunicationAnalyzerTime from msprof_analyze.test.st.cluster_analyse.cluster_step_trace_time_db import ClusterStepTraceTimeDb diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py index 17c9c29a04..6095475ab0 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py @@ -1,6 +1,7 @@ import unittest -from msprof_analyze.advisor.advisor_backend.prof_bean_advisor.cluster_step_trace_time_bean import ClusterStepTraceTimeBean +from msprof_analyze.advisor.advisor_backend.prof_bean_advisor.cluster_step_trace_time_bean \ + import ClusterStepTraceTimeBean class TestClusterStepTraceTimeBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py index fe09c5ffab..3f299b5a2c 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py @@ -1,7 +1,8 @@ import unittest from unittest.mock import patch -from msprof_analyze.compare_tools.compare_backend.comparator.operator_statistic_comparator import OperatorStatisticComparator +from msprof_analyze.compare_tools.compare_backend.comparator.operator_statistic_comparator \ + import OperatorStatisticComparator class MockBean: diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_compare_event.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_compare_event.py index 175292ec40..8b2c40dcd4 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_compare_event.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_compare_event.py @@ -1,6 +1,9 @@ import unittest -from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.compare_event import KernelEvent, MemoryEvent +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.compare_event import ( + KernelEvent, + MemoryEvent +) from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py index 30b46b903b..94ff68eceb 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py @@ -1,6 +1,7 @@ import unittest -from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.kernel_details_bean import KernelDetailsBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.kernel_details_bean \ + import KernelDetailsBean class TestKernelDetailsBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_memory_record_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_memory_record_bean.py index b32cb3299c..2488105bec 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_memory_record_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_memory_record_bean.py @@ -1,6 +1,7 @@ import unittest -from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.memory_record_bean import MemoryRecordBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.memory_record_bean \ + import MemoryRecordBean class TestMemoryRecordBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_operator_memory_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_operator_memory_bean.py index cb5cf450b1..750da0b9be 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_operator_memory_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_operator_memory_bean.py @@ -1,6 +1,7 @@ import unittest -from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.operator_memory_bean import OperatorMemoryBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.operator_memory_bean \ + import OperatorMemoryBean class TestOperatorMemoryBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py index 853e6a5ecf..9b8a53371b 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py @@ -2,7 +2,10 @@ import unittest from unittest.mock import patch from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import BaseProfilingParser, ProfilingResult +from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import ( + BaseProfilingParser, + ProfilingResult +) class ProfilingParser(BaseProfilingParser): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py index 69159aa2b9..07beb60698 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py @@ -1,7 +1,8 @@ import unittest from unittest.mock import patch -from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.operator_memory_bean import OperatorMemoryBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.operator_memory_bean \ + import OperatorMemoryBean from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import ProfilingResult from msprof_analyze.compare_tools.compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingParser -- Gitee From e911e334fbcd258453dbb61e70110d2290355fef Mon Sep 17 00:00:00 2001 From: hehongzhe <935062458@qq.com> Date: Wed, 5 Feb 2025 17:24:35 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=E7=BB=91=E6=A0=B8=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E7=A7=BB=E5=88=B0profiler=E7=9B=AE=E5=BD=95=E4=B8=8B=E5=AF=B9?= =?UTF-8?q?=E5=BA=94=E8=B5=84=E6=96=99=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- .../cluster_advice/cluster_advice_base.py | 2 +- .../dataset/profiling/profiling_dataset.py | 4 ++++ ...st_advisor_cmd_single_ascend_pt_compare.py | 2 +- .../advisor/common/test_enum_params_parser.py | 20 ++++++------------- 5 files changed, 13 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 4c706cc6fe..5ae0bf742f 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ 将 Ascend PyTorch Profiler 或者 msprof 采集的 PyTorch 场景性能数据进行分析,并输出性能调优建议。 -4. [bind_core](./profiler/msprof_analyze/affinity_cpu_bind) +4. [bind_core](./profiler/affinity_cpu_bind) 绑核脚本,支持非侵入修改工程代码,实现一键式绑核功能。 diff --git a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_advice_base.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_advice_base.py index 493947121c..5620f8e49e 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_advice_base.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_advice_base.py @@ -17,7 +17,7 @@ import os from abc import abstractmethod from msprof_analyze.advisor.advisor_backend.advice_base import AdviceBase -from cluster_analysis.interface import Interface +from msprof_analyze.cluster_analyse.cluster_analysis import Interface from msprof_analyze.advisor.advisor_backend.logger import Logger from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/advisor/dataset/profiling/profiling_dataset.py b/profiler/msprof_analyze/advisor/dataset/profiling/profiling_dataset.py index 65f70d69c6..7981e4140f 100644 --- a/profiler/msprof_analyze/advisor/dataset/profiling/profiling_dataset.py +++ b/profiler/msprof_analyze/advisor/dataset/profiling/profiling_dataset.py @@ -18,6 +18,10 @@ import os import yaml from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.advisor.common.profiling.ge_info import GeInfo +from msprof_analyze.advisor.common.profiling.msprof import Msprof +from msprof_analyze.advisor.common.profiling.op_summary import OpSummary +from msprof_analyze.advisor.common.profiling.tasktime import TaskTime from msprof_analyze.advisor.common.enum_params_parser import EnumParamsParser from msprof_analyze.advisor.dataset.dataset import Dataset from msprof_analyze.advisor.dataset.profiling.device_info import DeviceInfoParser diff --git a/profiler/msprof_analyze/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py b/profiler/msprof_analyze/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py index 487d504fd5..a485d62188 100644 --- a/profiler/msprof_analyze/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py +++ b/profiler/msprof_analyze/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py @@ -22,7 +22,7 @@ import pandas as pd from bs4 import BeautifulSoup from msprof_analyze.prof_common.path_manager import PathManager -from msprof_analyze.prof_common.utils import get_files, execute_cmd +from msprof_analyze.test.st.advisor.utils import get_files, execute_cmd class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/common/test_enum_params_parser.py b/profiler/msprof_analyze/test/ut/advisor/common/test_enum_params_parser.py index 5d11af1278..8f50972e46 100644 --- a/profiler/msprof_analyze/test/ut/advisor/common/test_enum_params_parser.py +++ b/profiler/msprof_analyze/test/ut/advisor/common/test_enum_params_parser.py @@ -1,18 +1,10 @@ -# Copyright (c) 2025, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. import unittest +import sys +import os + +profiler_root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) +sys.path.append(os.path.join(profiler_root_dir, "compare_tools")) +sys.path.append(os.path.join(profiler_root_dir, "cluster_analyse")) from msprof_analyze.advisor.common.enum_params_parser import EnumParamsParser from msprof_analyze.test.ut.advisor.advisor_backend.tools.tool import recover_env -- Gitee From d46f9ab4bd1fdebcbc9f89c458dd7157b1543da2 Mon Sep 17 00:00:00 2001 From: hehongzhe <935062458@qq.com> Date: Wed, 5 Feb 2025 20:31:32 +0800 Subject: [PATCH 5/7] bugfix --- .../msprof_analyze/prof_common/__init__.py | 5 +- profiler/msprof_analyze/prof_common/utils.py | 7 +++ .../test_cluster_advice_base.py | 6 +- .../advisor/common/test_enum_params_parser.py | 14 +++++ .../test_pytorch_data_preprocessor.py | 3 +- .../test_communication_group_generator.py | 12 ++-- .../compare_bean/test_memory_compare_bean.py | 9 ++- .../test_memory_statistic_bean.py | 6 +- .../test_operator_compare_bean.py | 9 ++- .../test_operator_statistic_bean.py | 6 +- .../test_base_profiling_parser.py | 27 ++++++--- .../test_gpu_profiling_parser.py | 36 +++++++---- .../test_npu_profiling_parser.py | 60 ++++++++++++------- .../ut/compare_tools/view/test_excel_view.py | 3 +- 14 files changed, 141 insertions(+), 62 deletions(-) diff --git a/profiler/msprof_analyze/prof_common/__init__.py b/profiler/msprof_analyze/prof_common/__init__.py index de0604079e..8b7e7544bb 100644 --- a/profiler/msprof_analyze/prof_common/__init__.py +++ b/profiler/msprof_analyze/prof_common/__init__.py @@ -11,4 +11,7 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. +import os +import sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) \ No newline at end of file diff --git a/profiler/msprof_analyze/prof_common/utils.py b/profiler/msprof_analyze/prof_common/utils.py index 9cfb438770..7794d6f985 100644 --- a/profiler/msprof_analyze/prof_common/utils.py +++ b/profiler/msprof_analyze/prof_common/utils.py @@ -12,6 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import sys +import os + +sys.path.append( + os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "cluster_analyse")) +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + import configparser import os from email.utils import parseaddr diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_advice_base.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_advice_base.py index c212396904..a86cd57af1 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_advice_base.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_advice_base.py @@ -70,7 +70,8 @@ class TestClusterAdviceBase(unittest.TestCase): def test_cluster_analyze_normal(self): mock_inst = MockChildClusterAdvice(self.tmp_dir) - with mock.patch("advisor_backend.cluster_advice.cluster_advice_base.Interface") as mock_if: + with mock.patch("msprof_analyze.advisor.advisor_backend.cluster_advice." + "cluster_advice_base.Interface") as mock_if: mock_if_inst = mock_if.return_value mock_if_inst.run = MagicMock(name="run") mock_inst.cluster_analyze() @@ -79,7 +80,8 @@ class TestClusterAdviceBase(unittest.TestCase): def test_cluster_analyze_abnormal(self): mock_inst = MockChildClusterAdvice(self.tmp_dir) with self.assertRaises(ValueError): - with mock.patch("advisor_backend.cluster_advice.cluster_advice_base.Interface") as mock_if: + with mock.patch("msprof_analyze.advisor.advisor_backend.cluster_advice." + "cluster_advice_base.Interface") as mock_if: mock_if_inst = mock_if.return_value mock_if_inst.run = mock.Mock(name="run", side_effect=Exception('Error!')) mock_inst.cluster_analyze() diff --git a/profiler/msprof_analyze/test/ut/advisor/common/test_enum_params_parser.py b/profiler/msprof_analyze/test/ut/advisor/common/test_enum_params_parser.py index 8f50972e46..88edc9424c 100644 --- a/profiler/msprof_analyze/test/ut/advisor/common/test_enum_params_parser.py +++ b/profiler/msprof_analyze/test/ut/advisor/common/test_enum_params_parser.py @@ -1,3 +1,17 @@ +# Copyright (c) 2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import unittest import sys import os diff --git a/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_pytorch_data_preprocessor.py b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_pytorch_data_preprocessor.py index 523c5032d6..ee159cca79 100644 --- a/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_pytorch_data_preprocessor.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_pytorch_data_preprocessor.py @@ -59,7 +59,8 @@ class TestPytorchDataPreprocessor(unittest.TestCase): def test_get_data_map_given_cluster_files_expect_rank_12(self): check = PytorchDataPreprocessor(self.dirs) - with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", return_value={}): + with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", + return_value={}): ret = check.get_data_map() self.assertIn(1, ret.keys()) self.assertIn(2, ret.keys()) diff --git a/profiler/msprof_analyze/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py b/profiler/msprof_analyze/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py index 743bf6fa6c..517327b811 100644 --- a/profiler/msprof_analyze/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py @@ -32,7 +32,8 @@ class TestCommunicationGroupGenerator(unittest.TestCase): check.collective_group_dict = { 'group1': {0} } - with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", return_value=True): + with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", + return_value=True): check.generate_p2p_communication_group() ret = {0} self.assertEqual(ret, set(check.communication_group[Constant.P2P][0])) @@ -43,7 +44,8 @@ class TestCommunicationGroupGenerator(unittest.TestCase): 'group1': {1, 2, 3, 4}, 'group2': {5, 6, 7, 8}, } - with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", return_value=True): + with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", + return_value=True): check.generate_p2p_communication_group() ret_a = {1, 2, 3, 4} ret_b = {5, 6, 7, 8} @@ -70,7 +72,8 @@ class TestCommunicationGroupGenerator(unittest.TestCase): 'group15': {15, 13}, 'group16': {15, 14} } - with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", return_value=True): + with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", + return_value=True): check.generate_p2p_communication_group() ret_a = {0, 1, 2, 3} ret_b = {4, 5, 6, 7} @@ -101,7 +104,8 @@ class TestCommunicationGroupGenerator(unittest.TestCase): 'group15': {12, 13, 14, 15}, 'group16': {12, 13, 14, 15} } - with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", return_value=True): + with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", + return_value=True): check.generate_p2p_communication_group() ret_a = {0, 1, 2, 3, 4, 5, 6, 7} ret_b = {8, 9, 10, 11, 12, 13, 14, 15} diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_compare_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_compare_bean.py index 5d8576872a..39efc1ac88 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_compare_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_compare_bean.py @@ -22,18 +22,21 @@ class TestMemoryCompareBean(unittest.TestCase): def test_row_when_valid_data(self): result = [2, self.name, None, None, 'add', 8, self.name, None, None, 'add', 8, 0, 1.0] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", return_value=[MockMemory(8)]): + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", + return_value=[MockMemory(8)]): mem = MemoryCompareBean(1, MockNode(self.name), MockNode(self.name)) self.assertEqual(mem.row, result) def test_row_when_invalid_base_data(self): result = [2, None, None, None, "", 0, self.name, None, None, 'add', 8, 8, float("inf")] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", return_value=[MockMemory(8)]): + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", + return_value=[MockMemory(8)]): mem = MemoryCompareBean(1, None, MockNode(self.name)) self.assertEqual(mem.row, result) def test_row_when_invalid_comparison_data(self): result = [2, self.name, None, None, 'add', 8, None, None, None, '', 0, -8, 0] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", return_value=[MockMemory(8)]): + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", + return_value=[MockMemory(8)]): mem = MemoryCompareBean(1, MockNode(self.name), None) self.assertEqual(mem.row, result) diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_statistic_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_statistic_bean.py index ae55d07121..48a4f0096b 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_statistic_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_statistic_bean.py @@ -15,21 +15,21 @@ class TestMemoryStatisticBean(unittest.TestCase): def test_row_when_valid_data(self): result = [None, self.name, 8.0, 40.0, 2, 4.0, 20.0, 1, -20.0, 0.5] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", return_value=[MockMemory(10240, 2000), MockMemory(10240, 2000)]): bean = MemoryStatisticBean(self.name, [1, 1], [1]) self.assertEqual(bean.row, result) def test_row_when_invalid_base_data(self): result = [None, self.name, 0, 0, 0, 4.0, 20.0, 1, 20.0, float("inf")] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", return_value=[MockMemory(10240, 2000), MockMemory(10240, 2000)]): bean = MemoryStatisticBean(self.name, [], [1]) self.assertEqual(bean.row, result) def test_row_when_invalid_comparison_data(self): result = [None, self.name, 8.0, 40.0, 2, 0, 0, 0, -40.0, 0] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", return_value=[MockMemory(10240, 2000), MockMemory(10240, 2000)]): bean = MemoryStatisticBean(self.name, [1, 1], []) self.assertEqual(bean.row, result) diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_compare_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_compare_bean.py index cc58d61c6d..15099a5f54 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_compare_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_compare_bean.py @@ -22,18 +22,21 @@ class TestOperatorCompareBean(unittest.TestCase): def test_row_when_valid_data(self): result = [2, self.name, None, None, 'add', 8, self.name, None, None, 'add', 8, 0, 1.0] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", return_value=[MockKernel(8)]): + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", + return_value=[MockKernel(8)]): op = OperatorCompareBean(1, MockNode(self.name), MockNode(self.name)) self.assertEqual(op.row, result) def test_row_when_invalid_base_data(self): result = [2, None, None, None, "", 0, self.name, None, None, 'add', 8, 8, float("inf")] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", return_value=[MockKernel(8)]): + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", + return_value=[MockKernel(8)]): op = OperatorCompareBean(1, None, MockNode(self.name)) self.assertEqual(op.row, result) def test_row_when_invalid_comparison_data(self): result = [2, self.name, None, None, 'add', 8, None, None, None, '', 0, -8, 0] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", return_value=[MockKernel(8)]): + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", + return_value=[MockKernel(8)]): op = OperatorCompareBean(1, MockNode(self.name), None) self.assertEqual(op.row, result) diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_statistic_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_statistic_bean.py index 9a4b70198e..5293d42846 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_statistic_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_statistic_bean.py @@ -14,21 +14,21 @@ class TestOperatorStatisticBean(unittest.TestCase): def test_row_when_valid_data(self): result = [None, self.name, 8.0, 2, 4.0, 1, -4.0, 0.5] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", return_value=[MockKernel(2000), MockKernel(2000)]): bean = OperatorStatisticBean(self.name, [1, 1], [1]) self.assertEqual(bean.row, result) def test_row_when_invalid_base_data(self): result = [None, self.name, 0, 0, 4.0, 1, 4.0, float("inf")] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", return_value=[MockKernel(2000), MockKernel(2000)]): bean = OperatorStatisticBean(self.name, [], [1]) self.assertEqual(bean.row, result) def test_row_when_invalid_comparison_data(self): result = [None, self.name, 8.0, 2, 0, 0, -8.0, 0] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", return_value=[MockKernel(2000), MockKernel(2000)]): bean = OperatorStatisticBean(self.name, [1, 1], []) self.assertEqual(bean.row, result) diff --git a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py index 9b8a53371b..de2ef0a468 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py @@ -103,55 +103,63 @@ class TestBaseProfilingParser(unittest.TestCase): def test_picking_torch_op_event(self): event = MockEvent(1, 2, 3) - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init({}, {}) self.assertTrue(parser._picking_torch_op_event(event)) def test_picking_kernel_event(self): event = MockEvent(1, 2, 3) - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init({}, {}) self.assertTrue(parser._picking_kernel_event(event)) def test_picking_flow_event(self): events = [MockEvent(1, 2, 3, "s"), MockEvent(1, 2, 3, "f")] - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init({}, {}) for event in events: self.assertTrue(parser._picking_flow_event(event)) def test_update_kernel_dict_when_valid_input(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init(self.flow_dict, self.all_kernels) parser._update_kernel_dict() self.assertEqual(len(parser._result_data.kernel_dict.get(12)), 2) def test_update_kernel_dict_when_without_kernels_return_null(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init(self.flow_dict, {}) parser._update_kernel_dict() self.assertEqual(len(parser._result_data.kernel_dict), 0) def test_update_kernel_dict_when_without_flow_return_null(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init({}, self.all_kernels) parser._update_kernel_dict() self.assertEqual(len(parser._result_data.kernel_dict), 0) def test_check_result_data(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init(self.flow_dict, self.all_kernels) parser._check_result_data() def test_load_data_when_valid_input(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init(self.flow_dict, self.all_kernels) result_data = parser.load_data() @@ -159,7 +167,8 @@ class TestBaseProfilingParser(unittest.TestCase): def test_update_communication_dict(self): result = {'allreduce': {'comm_list': [2.0], 'comm_task': {'notify_wait': [1.0]}}} - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init({}, {}) parser._comm_task_list = [TraceEventBean(event) for event in self.task_events] diff --git a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py index 767e078bcc..9e4d76bf97 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py @@ -52,8 +52,10 @@ class TestGpuProfilingParser(unittest.TestCase): other_event = {"ph": "X", "name": "other", "pid": 1, "tid": 1, "ts": 6, "dur": 1} def test_update_memory_list_when_valid_input(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.gpu_profiling_parser.GPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "gpu_profiling_parser.GPUProfilingParser.__init__", return_value=None): res = GPUProfilingParser({}, {}) res._enable_memory_compare = True @@ -64,8 +66,10 @@ class TestGpuProfilingParser(unittest.TestCase): self.assertEqual(res._result_data.memory_list[0].memory_details, ", (1, 2), [duration: 1.0], [size: 0.5]\n") def test_calculate_performance_time_when_valid_input(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.gpu_profiling_parser.GPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "gpu_profiling_parser.GPUProfilingParser.__init__", return_value=None): res = GPUProfilingParser({}, {}) res._profiling_type = "GPU" @@ -90,8 +94,10 @@ class TestGpuProfilingParser(unittest.TestCase): self.assertEqual(res._result_data.overall_metrics.compute_time, 7) def test_picking_memory_event_when_valid_input(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.gpu_profiling_parser.GPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "gpu_profiling_parser.GPUProfilingParser.__init__", return_value=None): res = GPUProfilingParser({}, {}) res._memory_events = [] @@ -102,8 +108,10 @@ class TestGpuProfilingParser(unittest.TestCase): def test_is_torch_op_event_when_valid_input(self): event_list = [{"cat": "cpu_op"}, {"cat": "user_annotation"}, {"cat": "cuda_runtime"}, {"cat": "operator"}] - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.gpu_profiling_parser.GPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "gpu_profiling_parser.GPUProfilingParser.__init__", return_value=None): res = GPUProfilingParser({}, {}) for event in event_list: @@ -115,8 +123,10 @@ class TestGpuProfilingParser(unittest.TestCase): def test_is_kernel_event_when_valid_input(self): event_list1 = [{"cat": "kernel", "name": "matmul"}, {"cat": "kernel", "name": "nccl_reduce"}] event_list2 = [{"cat": "async", "name": "nccl_reduce"}, {"cat": "cpu_op", "name": "aten::to"}] - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.gpu_profiling_parser.GPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "gpu_profiling_parser.GPUProfilingParser.__init__", return_value=None): res = GPUProfilingParser({}, {}) for event in event_list1: @@ -127,8 +137,10 @@ class TestGpuProfilingParser(unittest.TestCase): self.assertFalse(result) def test_is_flow_event_when_valid_input(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.gpu_profiling_parser.GPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "gpu_profiling_parser.GPUProfilingParser.__init__", return_value=None): res = GPUProfilingParser({}, {}) res._flow_cat = ("async_gpu",) diff --git a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py index 07beb60698..8f33065c16 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py @@ -22,8 +22,10 @@ class TestNPUProfilingParser(unittest.TestCase): {"ph": "M", "name": "thread_sort_index", "pid": 7, "tid": 3, "args": {"sort_index": 0}}] def test_update_memory_list_when_invalid_path(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None): res = NPUProfilingParser({}, {}) res._operator_memory_path = "" @@ -37,10 +39,13 @@ class TestNPUProfilingParser(unittest.TestCase): OperatorMemoryBean({"Name": "cann::add", "Size(KB)": 512, "Allocation Time(us)": 2, "Release Time(us)": 4}), OperatorMemoryBean( {"Name": "aten::add", "Size(KB)": 512, "Allocation Time(us)": 7, "Release Time(us)": 10})] - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None), \ - patch("msprof_analyze.prof_common.file_manager.FileManager.read_csv_file", return_value=memory_data): + patch("msprof_analyze.prof_common.file_manager.FileManager.read_csv_file", + return_value=memory_data): res = NPUProfilingParser({}, {}) res._path_level='' res._operator_memory_path = "" @@ -53,8 +58,10 @@ class TestNPUProfilingParser(unittest.TestCase): self.assertEqual(res._result_data.memory_list[0].duration, 2) def test_picking_hccl_event(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None): res = NPUProfilingParser({}, {}) res._hccl_pid = 7 @@ -68,8 +75,10 @@ class TestNPUProfilingParser(unittest.TestCase): self.assertEqual(len(res._comm_list), 1) def test_picking_task_queue_data(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None): res = NPUProfilingParser({}, {}) res._enqueue_dict = {} @@ -84,8 +93,10 @@ class TestNPUProfilingParser(unittest.TestCase): self.assertEqual(len(res._dequeue_data), 1) def test_picking_overlap_analysis_data(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None): res = NPUProfilingParser({}, {}) res._overlap_analysis = [] @@ -98,8 +109,10 @@ class TestNPUProfilingParser(unittest.TestCase): self.assertFalse(result) def test_is_kernel_event(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None): res = NPUProfilingParser({}, {}) res._kernel_pid = 5 @@ -108,27 +121,34 @@ class TestNPUProfilingParser(unittest.TestCase): self.assertFalse(res._is_kernel_event(TraceEventBean({"pid": 1, "ph": "x"}))) def test_is_flow_event(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None): res = NPUProfilingParser({}, {}) self.assertTrue(res._is_flow_event(TraceEventBean({"cat": "async_npu"}))) self.assertFalse(res._is_flow_event(TraceEventBean({"cat": "async"}))) def test_is_torch_op_event(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None): res = NPUProfilingParser({}, {}) self.assertTrue(res._is_torch_op_event(TraceEventBean({"cat": "cpu_op"}))) self.assertFalse(res._is_torch_op_event(TraceEventBean({"cat": "async"}))) def test_filter_meta_id(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None), \ patch( - "compare_backend.profiling_parser.npu_profiling_parser.BaseProfilingParser._trace_event_generator", + "compare_backend.profiling_parser.npu_profiling_parser.BaseProfilingParser." + "_trace_event_generator", return_value=(TraceEventBean(event) for event in self.meta_events)): res = NPUProfilingParser({}, {}) res._hccl_op_tid_list = [] diff --git a/profiler/msprof_analyze/test/ut/compare_tools/view/test_excel_view.py b/profiler/msprof_analyze/test/ut/compare_tools/view/test_excel_view.py index cabc23877c..20357a886b 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/view/test_excel_view.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/view/test_excel_view.py @@ -14,5 +14,6 @@ class TestExcelView(unittest.TestCase): os.remove(self.file_path) def test_generate_view(self): - with patch("compare_backend.view.work_sheet_creator.WorkSheetCreator.create_sheet"): + with patch("msprof_analyze.compare_tools.compare_backend.view.work_sheet_creator." + "WorkSheetCreator.create_sheet"): ExcelView({"table1": {}, "table2": {}}, self.file_path, {}).generate_view() -- Gitee From fd6f32fa4a6551215c78e1dda21229db60d8cb21 Mon Sep 17 00:00:00 2001 From: hehongzhe <935062458@qq.com> Date: Wed, 5 Feb 2025 22:12:56 +0800 Subject: [PATCH 6/7] merge --- .../cluster_prof_info_analysis.py | 345 ------------------ 1 file changed, 345 deletions(-) delete mode 100644 profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/cluster_prof_info_analysis.py diff --git a/profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/cluster_prof_info_analysis.py b/profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/cluster_prof_info_analysis.py deleted file mode 100644 index 78bf78ddfb..0000000000 --- a/profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/cluster_prof_info_analysis.py +++ /dev/null @@ -1,345 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import argparse -import re -import os -import stat -import warnings -from pathlib import Path - -import pandas as pd -import plotly.graph_objects as go -from plotly.subplots import make_subplots -from plotly.offline import plot - -from msprof_analyze.prof_common.logger import get_logger - -sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from msprof_analyze.prof_common.path_manager import PathManager -from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager - -logger = get_logger() - -MAX_READ_FILE_BYTES = 64 * 1024 * 1024 - - -class FormDataProcessor: - def __init__(self, path, form_name): - self.form_name = form_name - self.files = self.get_files_with_prefix_recursive(path, form_name) - - @staticmethod - def get_device_id(dir_path): - device_id = re.search(r'device_(\d+)', dir_path).group(1) - return device_id - - @staticmethod - def get_node_id(dir_path): - node_id = re.search(r'node(\d+)', dir_path).group(1) - return int(node_id) - - @staticmethod - def get_files_with_prefix_recursive(csv_path, match_str): - matched_ir_files = list(Path(csv_path).rglob(match_str)) - if not matched_ir_files: - msg = f"Didn't find any file in folder {csv_path} that matches {match_str}" - raise RuntimeError(msg) - return [str(item) for item in matched_ir_files] - - def read_summary_data(self, columns_to_keep): - # 存储所有合并后的数据 - all_data = pd.DataFrame() - for f in self.files: - if "mindstudio_profiler_output" in f: - continue - # 判断csv文件大小 - PathManager.check_file_size(f) - PathManager.check_path_readable(f) - # 读取CSV文件 - df = pd.read_csv(f) - # 保留需要的列 - try: - df = df[columns_to_keep] - except KeyError: - logger.info("%s文件没有所需的列,请确认profiling数据的正确性:\n," - "以下列可能不存在%s\n", f, columns_to_keep) - continue - # 从文件名提取设备ID - try: - df['device_id'] = self.get_device_id(f) - except Exception: - logger.info("文件 \"%s\" 的路径或者是文件夹名没有按照要求,请确保存在[device_]这一级文件夹," - "具体操作指导见readme\n", f) - continue - # 添加新列 "device_id" - try: - df['node_id'] = self.get_node_id(f) - except Exception: - logger.info("文件 \"%s\" 的路径或者是文件夹名没有按照要求,请确保存在[node*]这一级文件夹," - "具体操作指导见readme\n", f) - continue - # 将数据添加到最终的数据框中 - all_data = pd.concat([all_data, df]) - return all_data - - def get_chip_type(self): - file = self.files[0] - PathManager.check_file_size(file) - PathManager.check_path_readable(file) - df = pd.read_csv(file) - if 'aiv_time(us)' in df.columns: - return "ASCEND_NEW" - return "ASCEND_OTHER" - - def get_rank_num(self): - return len(self.files) - - -# 表驱动,获取不同芯片类型不同交付件的所需的列 -class ViewInfoManager: - def __init__(self, chip_type): - self.chip_type = chip_type - self.op_summary_columns_dict = {} - self.set_op_summary_columns_params() - - def set_op_summary_columns_params(self): - # 有些数据除了用表格的列进行分组之外,还添加了其他属性对数据进行分类,这部分数据放在extend_attr_to_group里面 - self.op_summary_columns_dict = { - 'ASCEND_NEW': { - 'TimeToCsvAnalyzer': - {'columns_to_group': ["Op Name", "Input Shapes", "Input Data Types", "Output Shapes"], - 'extend_attr_to_group': ["device_id", "node_id"], - 'columns_to_view': ["Task Duration(us)"], - 'calculate_fun': ['mean', 'var', 'max', 'min'] - }, - 'StatisticalInfoToHtmlAnalyzer': - {'columns_to_group': ["Op Name", "Input Shapes", "Input Data Types", "Output Shapes"], - "columns_to_view": ["Task Duration(us)", "aiv_time(us)", "aiv_vec_ratio", - "aiv_scalar_ratio", "aiv_mte2_ratio", "aiv_mte3_ratio", - "aicore_time(us)", "aic_mac_ratio", "aic_scalar_ratio", - "aic_mte1_ratio", "aic_mte2_ratio", "aic_fixpipe_ratio" - ], - 'calculate_fun': ['mean', 'var', 'max', 'min'] - } - }, - 'ASCEND_OTHER': { - 'TimeToCsvAnalyzer': - {'columns_to_group': ["Op Name", "Input Shapes", "Input Data Types", "Output Shapes"], - 'extend_attr_to_group': ["device_id", "node_id"], - "columns_to_view": ["Task Duration(us)"], - 'calculate_fun': ['mean', 'var', 'max', 'min'] - }, - 'StatisticalInfoToHtmlAnalyzer': - {'columns_to_group': ["Op Name", "Input Shapes", "Input Data Types", "Output Shapes"], - "columns_to_view": ["aicore_time(us)", "Task Duration(us)", "mac_ratio", "vec_ratio", - "scalar_ratio", "mte1_ratio", "mte2_ratio", "mte3_ratio"], - 'calculate_fun': ['mean', 'var', 'max', 'min'] - } - } - } - - def get_columns_info(self, analyzer_type): - return self.op_summary_columns_dict.get(self.chip_type, {}).get(analyzer_type) - - -class OpSummaryAnalyzerBase: - def __init__(self, chip_type, analyzer_type, dir_path): - self.chip_type = chip_type - view_info = ViewInfoManager(chip_type).get_columns_info(analyzer_type) - self.columns_to_view = view_info['columns_to_view'] - self.calculate_fun = view_info['calculate_fun'] - self.columns_to_group = view_info['columns_to_group'] - self.attrs_to_group = self.columns_to_group.copy() - if 'extend_attr_to_group' in view_info: - extend_attr_to_group = view_info['extend_attr_to_group'] - self.attrs_to_group.extend(extend_attr_to_group) - # 创建结果文件 - self.result_dir = os.path.join(dir_path, "result") - PathManager.check_path_length(self.result_dir) - PathManager.remove_path_safety(self.result_dir) - PathManager.check_path_writeable(dir_path) - PathManager.make_dir_safety(self.result_dir) - PathManager.check_path_writeable(self.result_dir) - - @staticmethod - def on_rm_error(func, path, exc_info): - # path contains the path of the file that couldn't be removed - # let's just assume that it's read-only and unlink it. - os.chmod(path, stat.S_IWRITE) - os.unlink(path) - - def get_columns_to_group(self): - return self.columns_to_group - - def get_columns_to_view(self): - return self.columns_to_view - - def calculate_view_data(self, summary_data): - # 存储所有合并后的数据 - calculate_dict = {self.columns_to_view[i]: self.calculate_fun for i in range(len(self.columns_to_view))} - view_data = summary_data.groupby(self.attrs_to_group).agg(calculate_dict).reset_index() - return view_data - - -class TimeToCsvAnalyzer(OpSummaryAnalyzerBase): - def __init__(self, chip_type, dir_path): - super().__init__(chip_type, "TimeToCsvAnalyzer", dir_path) - - def generate_deliverable(self, summary_data, rank_num): - view_data = self.calculate_view_data(summary_data) - # 规范化列名 - view_data.columns = [''.join(col) if col[1] == "" else '_'.join(col) for col in view_data.columns] - try: - for column in self.columns_to_view: - view_data[column + '_range'] = view_data[column + '_max'] - view_data[column + '_min'] - except Exception as e: - raise RuntimeError("Invalid view data!") from e - save_path = os.path.join(self.result_dir, "cluster_duration_time_analysis.csv") - PathManager.check_path_length(save_path) - view_data.to_csv(save_path, index=False) - # 该文件权限设置为只读权限,不允许修改 - os.chmod(save_path, stat.S_IROTH) - return view_data - - -class StatisticalInfoToHtmlAnalyzer(OpSummaryAnalyzerBase): - def __init__(self, chip_type, top_n, dir_path): - super().__init__(chip_type, "StatisticalInfoToHtmlAnalyzer", dir_path) - self.top_n = top_n - # top_n 如果不符合要求,报警告 - - def generate_deliverable(self, summary_data, rank_num): - view_data = self.calculate_view_data(summary_data) - # 规范化列名 op_name/ --> op_name time/var 这种不变 - view_data.columns = [''.join(col) if col[1] == "" else col for col in view_data.columns] - - # 对使用到的变量进行初始设置 - self.top_n = min(max(self.top_n, 1), len(view_data)) - top_n_data = view_data.sort_values(("Task Duration(us)", 'var'), ascending=False).head(self.top_n) - - for column in self.columns_to_view: - # 分别给每一种特性画图 - self.draw_plotly(column, summary_data, top_n_data, rank_num) - - def draw_plotly(self, column, summary_data, top_n_data, rank_num): - col_num = self.get_cal_num(rank_num) - row_num = self.top_n // col_num if self.top_n % col_num == 0 else (self.top_n + 1) // col_num - fig = make_subplots(rows=row_num, cols=col_num, vertical_spacing=0.03) - for i, (_, operation) in enumerate(top_n_data.iterrows()): - op_data = summary_data[(summary_data["Op Name"] == operation["Op Name"]) & - (summary_data["Input Shapes"] == operation["Input Shapes"]) & - (summary_data["Input Data Types"] == operation["Input Data Types"])] - op_data = op_data.sort_values(by=["node_id", "device_id"]) - node_ids = op_data['node_id'].unique() - device_ids = op_data['device_id'].unique() - - for node_id in node_ids: - for device_id in device_ids: - draw_data = op_data[(op_data['node_id'] == node_id) & (op_data['device_id'] == device_id)] - fig.add_trace(go.Box(y=draw_data[column], - name=f'{node_id}_{device_id}', - marker_color='green', showlegend=False), (i // col_num) + 1, (i % col_num) + 1) - - fig.update_xaxes(title_text=f'{operation["Op Name"]}-{operation["Input Shapes"]}', row=(i // col_num) + 1, - col=(i % col_num) + 1) - fig.update_layout(margin=dict(l=20, r=20, t=20, b=20), - height=int(500 * row_num), - width=int(rank_num * 100 * col_num), - title_text="Op Performance Comparison") - save_plot_path = os.path.join(self.result_dir, column + "_Info.html") - PathManager.check_path_length(save_plot_path) - plot(fig, filename=save_plot_path) - # 该文件权限设置为只读权限,不允许修改 - os.chmod(save_plot_path, stat.S_IROTH) - - def get_cal_num(self, rank_num): - # 计算每行应该画多少个子图 - if rank_num <= 16: - return 2 - else: - return 1 - - -class DeliverableGenerator: - def __init__(self, params): - self.dirs = params.get('dir') - self.form_process = FormDataProcessor(self.dirs, 'op_summary*.csv') - self.analyzers = [] - self.columns_to_keep = [] - self.set_analyzers(params) - self.set_columns_to_keep() - - def run(self): - summary_data = self.form_process.read_summary_data(self.columns_to_keep) - # 判断summarydata 数据是否为空,如果是空, 说明所有csv读取数据都失败了 - if summary_data.empty: - logger.info("没有符合要求的csv表格数据,请排查您的PROFILING数据") - return - rank_num = self.form_process.get_rank_num() - for analyzer in self.analyzers: - analyzer.generate_deliverable(summary_data, rank_num) - - def set_analyzers(self, params): - chip_type = self.form_process.get_chip_type() - # 判断该路径是不是软链接,并修改为绝对路径 - if os.path.islink(params.get('dir')): - logger.info("The file: \"%s\" is link. Please check the path.", params.get('dir')) - return - prof_path = os.path.abspath(params.get('dir')) - PathManager.input_path_common_check(prof_path) - if params.get('type') == "all": - self.analyzers = [TimeToCsvAnalyzer(chip_type, prof_path), - StatisticalInfoToHtmlAnalyzer(chip_type, params.get("top_n"), prof_path)] - elif params.get('type') == "html": - self.analyzers = [StatisticalInfoToHtmlAnalyzer(chip_type, params.get("top_n"), prof_path)] - elif params.get('type') == "csv": - self.analyzers = [TimeToCsvAnalyzer(chip_type, prof_path)] - else: - warnings.warn("参数错误,请输入 all html csv 这三种类型") # 发出一个警告信息 - - - def set_columns_to_keep(self): - columns_to_keep = [] - for analyzer in self.analyzers: - columns_to_keep.extend(analyzer.get_columns_to_group()) - columns_to_keep.extend(analyzer.get_columns_to_view()) - self.columns_to_keep = list(set(columns_to_keep)) - - -def main(): - # 解析命令行参数 - parser = argparse.ArgumentParser() - parser.add_argument("--dir", "-d", default=None, help="root dir of PROF_* data") - parser.add_argument("--top_n", "-n", default=10, help="how many operators to show", type=int) - parser.add_argument("--type", "-t", default='html', help="compare ratio or aicore-time", type=str) - parser.add_argument("--force", action='store_true', - help="Indicates whether to skip file size verification and owner verification") - args = parser.parse_args() - params = { - "dir": args.dir, - "top_n": args.top_n, - "type": args.type, - "force": args.force - } - AdditionalArgsManager().init(params) - deviverable_gen = DeliverableGenerator(params) - deviverable_gen.run() - -if __name__ == "__main__": - main() -- Gitee From b8ffa3efecc665df4b072058fa61d8d16850bdf2 Mon Sep 17 00:00:00 2001 From: hehongzhe <935062458@qq.com> Date: Wed, 5 Feb 2025 22:18:30 +0800 Subject: [PATCH 7/7] add file --- .../cluster_advice/cluster_pipeline_advice.py | 2 +- .../cluster_prof_info_analysis.py | 345 ++++++++++++++++++ .../test_cluster_pipeline_advice.py | 3 +- .../test_operator_statistic_comparator.py | 3 +- 4 files changed, 350 insertions(+), 3 deletions(-) create mode 100644 profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/cluster_prof_info_analysis.py diff --git a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py index 25436cfb5b..8db0f6fba4 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py @@ -27,11 +27,11 @@ from typing import Optional from typing import Tuple from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_advice_base import ClusterAdviceBase -from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant from msprof_analyze.advisor.advisor_backend.common_func_advisor.trace_view_preprocessor import FineTraceViewData from msprof_analyze.advisor.advisor_backend.common_func_advisor.trace_view_preprocessor import TraceViewPreProcessor from msprof_analyze.advisor.advisor_backend.logger import Logger +from msprof_analyze.cluster_analyse.cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor from msprof_analyze.prof_common.file_manager import FileManager logger = Logger() diff --git a/profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/cluster_prof_info_analysis.py b/profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/cluster_prof_info_analysis.py new file mode 100644 index 0000000000..78bf78ddfb --- /dev/null +++ b/profiler/msprof_analyze/cluster_analyse/cluster_kernels_analysis/cluster_prof_info_analysis.py @@ -0,0 +1,345 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import argparse +import re +import os +import stat +import warnings +from pathlib import Path + +import pandas as pd +import plotly.graph_objects as go +from plotly.subplots import make_subplots +from plotly.offline import plot + +from msprof_analyze.prof_common.logger import get_logger + +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from msprof_analyze.prof_common.path_manager import PathManager +from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager + +logger = get_logger() + +MAX_READ_FILE_BYTES = 64 * 1024 * 1024 + + +class FormDataProcessor: + def __init__(self, path, form_name): + self.form_name = form_name + self.files = self.get_files_with_prefix_recursive(path, form_name) + + @staticmethod + def get_device_id(dir_path): + device_id = re.search(r'device_(\d+)', dir_path).group(1) + return device_id + + @staticmethod + def get_node_id(dir_path): + node_id = re.search(r'node(\d+)', dir_path).group(1) + return int(node_id) + + @staticmethod + def get_files_with_prefix_recursive(csv_path, match_str): + matched_ir_files = list(Path(csv_path).rglob(match_str)) + if not matched_ir_files: + msg = f"Didn't find any file in folder {csv_path} that matches {match_str}" + raise RuntimeError(msg) + return [str(item) for item in matched_ir_files] + + def read_summary_data(self, columns_to_keep): + # 存储所有合并后的数据 + all_data = pd.DataFrame() + for f in self.files: + if "mindstudio_profiler_output" in f: + continue + # 判断csv文件大小 + PathManager.check_file_size(f) + PathManager.check_path_readable(f) + # 读取CSV文件 + df = pd.read_csv(f) + # 保留需要的列 + try: + df = df[columns_to_keep] + except KeyError: + logger.info("%s文件没有所需的列,请确认profiling数据的正确性:\n," + "以下列可能不存在%s\n", f, columns_to_keep) + continue + # 从文件名提取设备ID + try: + df['device_id'] = self.get_device_id(f) + except Exception: + logger.info("文件 \"%s\" 的路径或者是文件夹名没有按照要求,请确保存在[device_]这一级文件夹," + "具体操作指导见readme\n", f) + continue + # 添加新列 "device_id" + try: + df['node_id'] = self.get_node_id(f) + except Exception: + logger.info("文件 \"%s\" 的路径或者是文件夹名没有按照要求,请确保存在[node*]这一级文件夹," + "具体操作指导见readme\n", f) + continue + # 将数据添加到最终的数据框中 + all_data = pd.concat([all_data, df]) + return all_data + + def get_chip_type(self): + file = self.files[0] + PathManager.check_file_size(file) + PathManager.check_path_readable(file) + df = pd.read_csv(file) + if 'aiv_time(us)' in df.columns: + return "ASCEND_NEW" + return "ASCEND_OTHER" + + def get_rank_num(self): + return len(self.files) + + +# 表驱动,获取不同芯片类型不同交付件的所需的列 +class ViewInfoManager: + def __init__(self, chip_type): + self.chip_type = chip_type + self.op_summary_columns_dict = {} + self.set_op_summary_columns_params() + + def set_op_summary_columns_params(self): + # 有些数据除了用表格的列进行分组之外,还添加了其他属性对数据进行分类,这部分数据放在extend_attr_to_group里面 + self.op_summary_columns_dict = { + 'ASCEND_NEW': { + 'TimeToCsvAnalyzer': + {'columns_to_group': ["Op Name", "Input Shapes", "Input Data Types", "Output Shapes"], + 'extend_attr_to_group': ["device_id", "node_id"], + 'columns_to_view': ["Task Duration(us)"], + 'calculate_fun': ['mean', 'var', 'max', 'min'] + }, + 'StatisticalInfoToHtmlAnalyzer': + {'columns_to_group': ["Op Name", "Input Shapes", "Input Data Types", "Output Shapes"], + "columns_to_view": ["Task Duration(us)", "aiv_time(us)", "aiv_vec_ratio", + "aiv_scalar_ratio", "aiv_mte2_ratio", "aiv_mte3_ratio", + "aicore_time(us)", "aic_mac_ratio", "aic_scalar_ratio", + "aic_mte1_ratio", "aic_mte2_ratio", "aic_fixpipe_ratio" + ], + 'calculate_fun': ['mean', 'var', 'max', 'min'] + } + }, + 'ASCEND_OTHER': { + 'TimeToCsvAnalyzer': + {'columns_to_group': ["Op Name", "Input Shapes", "Input Data Types", "Output Shapes"], + 'extend_attr_to_group': ["device_id", "node_id"], + "columns_to_view": ["Task Duration(us)"], + 'calculate_fun': ['mean', 'var', 'max', 'min'] + }, + 'StatisticalInfoToHtmlAnalyzer': + {'columns_to_group': ["Op Name", "Input Shapes", "Input Data Types", "Output Shapes"], + "columns_to_view": ["aicore_time(us)", "Task Duration(us)", "mac_ratio", "vec_ratio", + "scalar_ratio", "mte1_ratio", "mte2_ratio", "mte3_ratio"], + 'calculate_fun': ['mean', 'var', 'max', 'min'] + } + } + } + + def get_columns_info(self, analyzer_type): + return self.op_summary_columns_dict.get(self.chip_type, {}).get(analyzer_type) + + +class OpSummaryAnalyzerBase: + def __init__(self, chip_type, analyzer_type, dir_path): + self.chip_type = chip_type + view_info = ViewInfoManager(chip_type).get_columns_info(analyzer_type) + self.columns_to_view = view_info['columns_to_view'] + self.calculate_fun = view_info['calculate_fun'] + self.columns_to_group = view_info['columns_to_group'] + self.attrs_to_group = self.columns_to_group.copy() + if 'extend_attr_to_group' in view_info: + extend_attr_to_group = view_info['extend_attr_to_group'] + self.attrs_to_group.extend(extend_attr_to_group) + # 创建结果文件 + self.result_dir = os.path.join(dir_path, "result") + PathManager.check_path_length(self.result_dir) + PathManager.remove_path_safety(self.result_dir) + PathManager.check_path_writeable(dir_path) + PathManager.make_dir_safety(self.result_dir) + PathManager.check_path_writeable(self.result_dir) + + @staticmethod + def on_rm_error(func, path, exc_info): + # path contains the path of the file that couldn't be removed + # let's just assume that it's read-only and unlink it. + os.chmod(path, stat.S_IWRITE) + os.unlink(path) + + def get_columns_to_group(self): + return self.columns_to_group + + def get_columns_to_view(self): + return self.columns_to_view + + def calculate_view_data(self, summary_data): + # 存储所有合并后的数据 + calculate_dict = {self.columns_to_view[i]: self.calculate_fun for i in range(len(self.columns_to_view))} + view_data = summary_data.groupby(self.attrs_to_group).agg(calculate_dict).reset_index() + return view_data + + +class TimeToCsvAnalyzer(OpSummaryAnalyzerBase): + def __init__(self, chip_type, dir_path): + super().__init__(chip_type, "TimeToCsvAnalyzer", dir_path) + + def generate_deliverable(self, summary_data, rank_num): + view_data = self.calculate_view_data(summary_data) + # 规范化列名 + view_data.columns = [''.join(col) if col[1] == "" else '_'.join(col) for col in view_data.columns] + try: + for column in self.columns_to_view: + view_data[column + '_range'] = view_data[column + '_max'] - view_data[column + '_min'] + except Exception as e: + raise RuntimeError("Invalid view data!") from e + save_path = os.path.join(self.result_dir, "cluster_duration_time_analysis.csv") + PathManager.check_path_length(save_path) + view_data.to_csv(save_path, index=False) + # 该文件权限设置为只读权限,不允许修改 + os.chmod(save_path, stat.S_IROTH) + return view_data + + +class StatisticalInfoToHtmlAnalyzer(OpSummaryAnalyzerBase): + def __init__(self, chip_type, top_n, dir_path): + super().__init__(chip_type, "StatisticalInfoToHtmlAnalyzer", dir_path) + self.top_n = top_n + # top_n 如果不符合要求,报警告 + + def generate_deliverable(self, summary_data, rank_num): + view_data = self.calculate_view_data(summary_data) + # 规范化列名 op_name/ --> op_name time/var 这种不变 + view_data.columns = [''.join(col) if col[1] == "" else col for col in view_data.columns] + + # 对使用到的变量进行初始设置 + self.top_n = min(max(self.top_n, 1), len(view_data)) + top_n_data = view_data.sort_values(("Task Duration(us)", 'var'), ascending=False).head(self.top_n) + + for column in self.columns_to_view: + # 分别给每一种特性画图 + self.draw_plotly(column, summary_data, top_n_data, rank_num) + + def draw_plotly(self, column, summary_data, top_n_data, rank_num): + col_num = self.get_cal_num(rank_num) + row_num = self.top_n // col_num if self.top_n % col_num == 0 else (self.top_n + 1) // col_num + fig = make_subplots(rows=row_num, cols=col_num, vertical_spacing=0.03) + for i, (_, operation) in enumerate(top_n_data.iterrows()): + op_data = summary_data[(summary_data["Op Name"] == operation["Op Name"]) & + (summary_data["Input Shapes"] == operation["Input Shapes"]) & + (summary_data["Input Data Types"] == operation["Input Data Types"])] + op_data = op_data.sort_values(by=["node_id", "device_id"]) + node_ids = op_data['node_id'].unique() + device_ids = op_data['device_id'].unique() + + for node_id in node_ids: + for device_id in device_ids: + draw_data = op_data[(op_data['node_id'] == node_id) & (op_data['device_id'] == device_id)] + fig.add_trace(go.Box(y=draw_data[column], + name=f'{node_id}_{device_id}', + marker_color='green', showlegend=False), (i // col_num) + 1, (i % col_num) + 1) + + fig.update_xaxes(title_text=f'{operation["Op Name"]}-{operation["Input Shapes"]}', row=(i // col_num) + 1, + col=(i % col_num) + 1) + fig.update_layout(margin=dict(l=20, r=20, t=20, b=20), + height=int(500 * row_num), + width=int(rank_num * 100 * col_num), + title_text="Op Performance Comparison") + save_plot_path = os.path.join(self.result_dir, column + "_Info.html") + PathManager.check_path_length(save_plot_path) + plot(fig, filename=save_plot_path) + # 该文件权限设置为只读权限,不允许修改 + os.chmod(save_plot_path, stat.S_IROTH) + + def get_cal_num(self, rank_num): + # 计算每行应该画多少个子图 + if rank_num <= 16: + return 2 + else: + return 1 + + +class DeliverableGenerator: + def __init__(self, params): + self.dirs = params.get('dir') + self.form_process = FormDataProcessor(self.dirs, 'op_summary*.csv') + self.analyzers = [] + self.columns_to_keep = [] + self.set_analyzers(params) + self.set_columns_to_keep() + + def run(self): + summary_data = self.form_process.read_summary_data(self.columns_to_keep) + # 判断summarydata 数据是否为空,如果是空, 说明所有csv读取数据都失败了 + if summary_data.empty: + logger.info("没有符合要求的csv表格数据,请排查您的PROFILING数据") + return + rank_num = self.form_process.get_rank_num() + for analyzer in self.analyzers: + analyzer.generate_deliverable(summary_data, rank_num) + + def set_analyzers(self, params): + chip_type = self.form_process.get_chip_type() + # 判断该路径是不是软链接,并修改为绝对路径 + if os.path.islink(params.get('dir')): + logger.info("The file: \"%s\" is link. Please check the path.", params.get('dir')) + return + prof_path = os.path.abspath(params.get('dir')) + PathManager.input_path_common_check(prof_path) + if params.get('type') == "all": + self.analyzers = [TimeToCsvAnalyzer(chip_type, prof_path), + StatisticalInfoToHtmlAnalyzer(chip_type, params.get("top_n"), prof_path)] + elif params.get('type') == "html": + self.analyzers = [StatisticalInfoToHtmlAnalyzer(chip_type, params.get("top_n"), prof_path)] + elif params.get('type') == "csv": + self.analyzers = [TimeToCsvAnalyzer(chip_type, prof_path)] + else: + warnings.warn("参数错误,请输入 all html csv 这三种类型") # 发出一个警告信息 + + + def set_columns_to_keep(self): + columns_to_keep = [] + for analyzer in self.analyzers: + columns_to_keep.extend(analyzer.get_columns_to_group()) + columns_to_keep.extend(analyzer.get_columns_to_view()) + self.columns_to_keep = list(set(columns_to_keep)) + + +def main(): + # 解析命令行参数 + parser = argparse.ArgumentParser() + parser.add_argument("--dir", "-d", default=None, help="root dir of PROF_* data") + parser.add_argument("--top_n", "-n", default=10, help="how many operators to show", type=int) + parser.add_argument("--type", "-t", default='html', help="compare ratio or aicore-time", type=str) + parser.add_argument("--force", action='store_true', + help="Indicates whether to skip file size verification and owner verification") + args = parser.parse_args() + params = { + "dir": args.dir, + "top_n": args.top_n, + "type": args.type, + "force": args.force + } + AdditionalArgsManager().init(params) + deviverable_gen = DeliverableGenerator(params) + deviverable_gen.run() + +if __name__ == "__main__": + main() diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py index 31992c4148..8f6d2a6a50 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py @@ -119,7 +119,8 @@ class TestClusterPipelineAdvice(unittest.TestCase): bp_op2 = {"ph": "X", "name": "autogard::add", "ts": str(2000000000 - 100), "dur": 2000, "tid": 3, "pid": 1, "args": {}} res_bp_ops = [(bp_op1, bp_op2)] - with mock.patch('advisor_backend.cluster_advice.cluster_pipeline_advice.ClusterPipelineAdvice.double_queue_pop', + with mock.patch('msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_pipeline_advice.' + 'ClusterPipelineAdvice.double_queue_pop', return_value=(None, res_bp_ops)): advice = ClusterPipelineAdvice('./tmp_dir', {}) res_check = advice.get_fp_bp_bound_ops(fine_data) diff --git a/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py index 3f299b5a2c..a123a98bff 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py @@ -25,7 +25,8 @@ class TestOperatorStatisticComparator(unittest.TestCase): base_dict = {"add": [1], "matmul": [1]} comparison_dict = {"add": [1], "reduce": [1]} with patch( - "compare_backend.comparator.operator_statistic_comparator.OperatorStatisticComparator._group_by_op_name", + "msprof_analyze.compare_tools.compare_backend.comparator.operator_statistic_comparator." + "OperatorStatisticComparator._group_by_op_name", return_value=(base_dict, comparison_dict)): comparator = OperatorStatisticComparator({1: 2}, MockBean) comparator._compare() -- Gitee