diff --git a/README.md b/README.md index 064720548f16342e8ad239420411a3cd2079be28..5ae0bf742fced7ed86452d03d013670cc3528316 100644 --- a/README.md +++ b/README.md @@ -30,17 +30,17 @@ [MindStudio Probe（msprobe，MindStudio 精度调试工具）](./debug/accuracy_tools/msprobe)。 -## [性能工具](./profiler) +## [性能工具](./profiler/msprof_analyze) -1. [compare_tools（性能比对工具）](./profiler/compare_tools) +1. [compare_tools（性能比对工具）](./profiler/msprof_analyze/compare_tools) 提供 NPU 与 GPU 性能拆解功能以及算子、通信、内存性能的比对功能。 -2. [cluster_analyse（集群分析工具）](./profiler/cluster_analyse) +2. [cluster_analyse（集群分析工具）](./profiler/msprof_analyze/cluster_analyse) 提供多机多卡的集群分析能力（基于通信域的通信分析和迭代耗时分析）, 当前需要配合 MindStudio Insight 的集群分析功能使用。 -3. [advisor](./profiler/advisor) +3. [advisor](./profiler/msprof_analyze/advisor) 将 Ascend PyTorch Profiler 或者 msprof 采集的 PyTorch 场景性能数据进行分析，并输出性能调优建议。 diff --git a/profiler/msprof_analyze/affinity_cpu_bind/README.md b/profiler/affinity_cpu_bind/README.md similarity index 100% rename from profiler/msprof_analyze/affinity_cpu_bind/README.md rename to profiler/affinity_cpu_bind/README.md diff --git a/profiler/msprof_analyze/affinity_cpu_bind/bind_core.py b/profiler/affinity_cpu_bind/bind_core.py similarity index 100% rename from profiler/msprof_analyze/affinity_cpu_bind/bind_core.py rename to profiler/affinity_cpu_bind/bind_core.py diff --git a/profiler/msprof_analyze/example/__init__.py b/profiler/example/__init__.py similarity index 100% rename from profiler/msprof_analyze/example/__init__.py rename to profiler/example/__init__.py diff --git a/profiler/msprof_analyze/example/mstx_torch_plugin/README.md b/profiler/example/mstx_torch_plugin/README.md similarity index 100% rename from profiler/msprof_analyze/example/mstx_torch_plugin/README.md rename to profiler/example/mstx_torch_plugin/README.md diff --git a/profiler/msprof_analyze/example/mstx_torch_plugin/__init__.py b/profiler/example/mstx_torch_plugin/__init__.py similarity index 100% rename from profiler/msprof_analyze/example/mstx_torch_plugin/__init__.py rename to profiler/example/mstx_torch_plugin/__init__.py diff --git a/profiler/msprof_analyze/example/mstx_torch_plugin/img/result.png b/profiler/example/mstx_torch_plugin/img/result.png similarity index 100% rename from profiler/msprof_analyze/example/mstx_torch_plugin/img/result.png rename to profiler/example/mstx_torch_plugin/img/result.png diff --git a/profiler/msprof_analyze/example/mstx_torch_plugin/mstx_torch_plugin.py b/profiler/example/mstx_torch_plugin/mstx_torch_plugin.py similarity index 100% rename from profiler/msprof_analyze/example/mstx_torch_plugin/mstx_torch_plugin.py rename to profiler/example/mstx_torch_plugin/mstx_torch_plugin.py diff --git a/profiler/msprof_analyze/example/setup.py b/profiler/example/setup.py similarity index 100% rename from profiler/msprof_analyze/example/setup.py rename to profiler/example/setup.py diff --git a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/cluster_advice_factory.py b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/cluster_advice_factory.py index 6bb93f46704eb13fef14d070f891e350446829ea..c3575c0a7576bce190c6b38aa31d32924dfb8e38 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/cluster_advice_factory.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/cluster_advice_factory.py @@ -12,12 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from advice_factory.advice_factory import AdviceFactory -from cluster_advice.slow_link_advice import SlowLinkAdvice -from cluster_advice.slow_rank_advice import SlowRankAdvice -from cluster_advice.cluster_pipeline_advice import ClusterPipelineAdvice -from cluster_advice.kernel_cluster_advice import KernelClusterAdvice -from common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.advice_factory.advice_factory import AdviceFactory +from msprof_analyze.advisor.advisor_backend.cluster_advice.slow_link_advice import SlowLinkAdvice +from msprof_analyze.advisor.advisor_backend.cluster_advice.slow_rank_advice import SlowRankAdvice +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_pipeline_advice import ClusterPipelineAdvice +from msprof_analyze.advisor.advisor_backend.cluster_advice.kernel_cluster_advice import KernelClusterAdvice +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant class ClusterAdviceFactory(AdviceFactory): diff --git a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/compute_advice_factory.py b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/compute_advice_factory.py index 336bef7dd8553eb82586d52260443a7d01e84ab0..026ea1675a1a923cc2c4328f492601e8f3a7c15d 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/compute_advice_factory.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/compute_advice_factory.py @@ -12,10 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from common_func_advisor.constant import Constant -from advice_factory.advice_factory import AdviceFactory -from compute_advice.npu_fused_advice import NpuFusedAdvice -from compute_advice.npu_slow_advice import NpuSlowAdvice +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.advice_factory.advice_factory import AdviceFactory +from msprof_analyze.advisor.advisor_backend.compute_advice.npu_fused_advice import NpuFusedAdvice +from msprof_analyze.advisor.advisor_backend.compute_advice.npu_slow_advice import NpuSlowAdvice class ComputeAdviceFactory(AdviceFactory): diff --git a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/overall_advice_factory.py b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/overall_advice_factory.py index baf80cc200f4c3cd1057b7fc28e750948a450cf1..9f4964aad4b303dcf071765eabf4b625f652b190 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/overall_advice_factory.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/overall_advice_factory.py @@ -12,9 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from advice_factory.advice_factory import AdviceFactory -from common_func_advisor.constant import Constant -from overall_advice.overall_summary_advice import OverallSummaryAdvice +from msprof_analyze.advisor.advisor_backend.advice_factory.advice_factory import AdviceFactory +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.overall_advice.overall_summary_advice import OverallSummaryAdvice class OverallAdviceFactory(AdviceFactory): diff --git a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/timeline_advice_factory.py b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/timeline_advice_factory.py index 44b352e95a7bb1007bc7373193603c2a0b9d8b6c..2051c019b19e2228c78f60b08317e6651f9c7ccd 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/timeline_advice_factory.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/advice_factory/timeline_advice_factory.py @@ -12,10 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from advice_factory.advice_factory import AdviceFactory -from common_func_advisor.constant import Constant -from timeline_advice.optimizer_advice import OptimizerAdvice -from timeline_advice.op_schedule_advice import OpScheduleAdvice +from msprof_analyze.advisor.advisor_backend.advice_factory.advice_factory import AdviceFactory +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.timeline_advice.optimizer_advice import OptimizerAdvice +from msprof_analyze.advisor.advisor_backend.timeline_advice.op_schedule_advice import OpScheduleAdvice class TimelineAdviceFactory(AdviceFactory): diff --git a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_advice_base.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_advice_base.py index d5013fffb4bad267b8401f6cb8e135eb91ff2ba6..5620f8e49ec54ccaad9f185fabe15d279e4c48f1 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_advice_base.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_advice_base.py @@ -16,9 +16,8 @@ import os from abc import abstractmethod -from advice_base import AdviceBase -from cluster_analysis import Interface - +from msprof_analyze.advisor.advisor_backend.advice_base import AdviceBase +from msprof_analyze.cluster_analyse.cluster_analysis import Interface from msprof_analyze.advisor.advisor_backend.logger import Logger from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py index 81fcf88ec098ce9dc958b4e387a6558c02f08b96..8db0f6fba4dc1ffc92faf1062f93add29b0dd11b 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/cluster_pipeline_advice.py @@ -26,13 +26,12 @@ from typing import List from typing import Optional from typing import Tuple -from cluster_advice.cluster_advice_base import ClusterAdviceBase -from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor -from common_func_advisor.constant import Constant -from common_func_advisor.trace_view_preprocessor import FineTraceViewData -from common_func_advisor.trace_view_preprocessor import TraceViewPreProcessor - +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_advice_base import ClusterAdviceBase +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.common_func_advisor.trace_view_preprocessor import FineTraceViewData +from msprof_analyze.advisor.advisor_backend.common_func_advisor.trace_view_preprocessor import TraceViewPreProcessor from msprof_analyze.advisor.advisor_backend.logger import Logger +from msprof_analyze.cluster_analyse.cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor from msprof_analyze.prof_common.file_manager import FileManager logger = Logger() diff --git a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py index e3b264463b2e93bf2a0381915ecd89b03a3b1dda..a7d3a010959275ee0fd6e3be2af926f7fb46c3bb 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py @@ -15,8 +15,8 @@ import os import pandas as pd -from common_func_advisor.constant import Constant as AdvisorConstant -from cluster_advice.cluster_advice_base import ClusterAdviceBase +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant as AdvisorConstant +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_advice_base import ClusterAdviceBase from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor from msprof_analyze.prof_common.file_manager import FileManager from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_link_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_link_advice.py index 564a844df3ab00e74a5021ef72fb0b55ea5aa107..2024adf8f6a020a5e09ce41949f9815831d7b563 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_link_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_link_advice.py @@ -15,8 +15,8 @@ import os from collections import defaultdict -from common_func_advisor.constant import Constant -from cluster_advice.cluster_advice_base import ClusterAdviceBase +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_advice_base import ClusterAdviceBase from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_rank_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_rank_advice.py index af644ed4b8822aef027fdf8f72078ebe1d6cefe6..182249e3a08a1e50d1178dbed9b6b17257fcbfd3 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_rank_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/cluster_advice/slow_rank_advice.py @@ -15,9 +15,10 @@ import os from collections import defaultdict -from common_func_advisor.constant import Constant -from cluster_advice.cluster_advice_base import ClusterAdviceBase -from prof_bean_advisor.cluster_step_trace_time_bean import ClusterStepTraceTimeBean +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_advice_base import ClusterAdviceBase +from msprof_analyze.advisor.advisor_backend.prof_bean_advisor.cluster_step_trace_time_bean \ + import ClusterStepTraceTimeBean from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_json.py b/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_json.py index 74152d628bb1b3ba0a54e48760630eb3afa1c52f..5af97c785aa431f32056078dd6868b0c954f40c9 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_json.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_json.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - from abc import abstractmethod from dataclasses import dataclass from dataclasses import field diff --git a/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py b/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py index 028ee8d4b14ab008cc8d43b45a93e026e82beeb1..782cf459eda6e471526d8cae924e1fecc272a1f2 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py @@ -19,7 +19,7 @@ import sys from typing import Optional from dataclasses import dataclass -from common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant from msprof_analyze.advisor.advisor_backend.logger import Logger logger = Logger() diff --git a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/compute_advice_base.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/compute_advice_base.py index 67e1513d81e271573cd919b2b34c13f1ee1b8fac..a137b832e040a4036b122d69f9aa1bc6e85e0d29 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/compute_advice_base.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/compute_advice_base.py @@ -18,7 +18,7 @@ from collections import defaultdict import os import logging -from advice_base import AdviceBase +from msprof_analyze.advisor.advisor_backend.advice_base import AdviceBase from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py index 1c7634a6e4b07924915652a4225e8aeec1280424..018c3c964db11cbe550c4fe1bb857f8b93ed3dc6 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py @@ -13,14 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import multiprocessing - import pandas as pd -import numpy as np from common_func.path_manager import PathManager -from common_func_advisor.constant import Constant -from .op_perf import OpPerfFactory +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant class CSVAnalyzer: diff --git a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/json_analyzer.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/json_analyzer.py index 8a32e6d14836e8e849b67c92a1b83c21d9db9078..d47d6d8ad34352edab2a2539dcb5b9a79580a399 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/json_analyzer.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/json_analyzer.py @@ -17,7 +17,7 @@ import logging import pandas as pd -from common_func_advisor.trace_view_json import TraceViewJson +from msprof_analyze.advisor.advisor_backend.common_func_advisor.trace_view_json import TraceViewJson logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/op_perf.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/op_perf.py index 57488ae0f6ad085e0f176cd97a0713baaa98dfb6..b8f5ef42850d98fb5fc44acaf5289b0c61ab84d0 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/op_perf.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused/op_perf.py @@ -16,9 +16,9 @@ import functools from typing import Dict import logging -from common_func_advisor.constant import Constant -from common_func_advisor.constant import CoreType -from common_func_advisor.constant import PerfColor +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import CoreType +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import PerfColor logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused_advice.py index a8a744226411e0fc1f2d66a28d61d11655b20106..349ede99059ffa98e0fb52888679e01cb15d2917 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_fused_advice.py @@ -13,15 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import logging from abc import ABC import pandas as pd -from compute_advice.compute_advice_base import ComputeAdviceBase -from compute_advice.npu_fused.csv_analyzer import CSVAnalyzer -from compute_advice.npu_fused.json_analyzer import JSONAnalyzer +from msprof_analyze.advisor.advisor_backend.compute_advice.compute_advice_base import ComputeAdviceBase +from msprof_analyze.advisor.advisor_backend.compute_advice.npu_fused.csv_analyzer import CSVAnalyzer +from msprof_analyze.advisor.advisor_backend.compute_advice.npu_fused.json_analyzer import JSONAnalyzer logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py index fc271e035e97d8eda74a00b125c93f5b8e98dfee..1eff2f6e9931748545e12ef5b6a5144a31fbe53d 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/compute_advice/npu_slow_advice.py @@ -20,11 +20,10 @@ import logging import pandas as pd from common_func.path_manager import PathManager -from compute_advice.compute_advice_base import ComputeAdviceBase -from compute_advice.npu_fused.op_perf import OpPerfFactory -from common_func_advisor.constant import Constant -from common_func_advisor.constant import PerfColor -from advisor_backend.common_func_advisor.trace_view_json import TraceViewJson +from msprof_analyze.advisor.advisor_backend.compute_advice.compute_advice_base import ComputeAdviceBase +from msprof_analyze.advisor.advisor_backend.compute_advice.npu_fused.op_perf import OpPerfFactory +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant, PerfColor +from msprof_analyze.advisor.advisor_backend.common_func_advisor.trace_view_json import TraceViewJson class NpuSlowAdvice(ComputeAdviceBase, ABC): diff --git a/profiler/msprof_analyze/advisor/advisor_backend/interface.py b/profiler/msprof_analyze/advisor/advisor_backend/interface.py index deb68822ec4ac025e6cf647cd031415618cc415e..f4e84f64e5afb2f94225eccd1f9e7a0dd0091589 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/interface.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/interface.py @@ -13,19 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -import sys - -sys.path.append( - os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "advisor_backend")) -sys.path.append( - os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "compare_tools")) -sys.path.append( - os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "cluster_analyse")) -from common_func_advisor.constant import Constant -from advisor_backend.advice_factory.cluster_advice_factory import ClusterAdviceFactory -from advisor_backend.advice_factory.compute_advice_factory import ComputeAdviceFactory -from advisor_backend.advice_factory.timeline_advice_factory import TimelineAdviceFactory -from advisor_backend.advice_factory.overall_advice_factory import OverallAdviceFactory + +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.advice_factory.cluster_advice_factory import ClusterAdviceFactory +from msprof_analyze.advisor.advisor_backend.advice_factory.compute_advice_factory import ComputeAdviceFactory +from msprof_analyze.advisor.advisor_backend.advice_factory.timeline_advice_factory import TimelineAdviceFactory +from msprof_analyze.advisor.advisor_backend.advice_factory.overall_advice_factory import OverallAdviceFactory class Interface: diff --git a/profiler/msprof_analyze/advisor/advisor_backend/overall_advice/overall_summary_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/overall_advice/overall_summary_advice.py index 407f636335d8402a511f29403beb74c910dbbd11..979fcf7246eccafda9fbb1d9e3565ac89b9b7420 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/overall_advice/overall_summary_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/overall_advice/overall_summary_advice.py @@ -15,11 +15,9 @@ import os import logging -from compare_interface.comparison_interface import ComparisonInterface -from advisor_backend.advice_base import AdviceBase - +from msprof_analyze.compare_tools.compare_interface.comparison_interface import ComparisonInterface +from msprof_analyze.advisor.advisor_backend.advice_base import AdviceBase from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt -from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager from msprof_analyze.prof_common.constant import Constant logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/op_schedule_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/op_schedule_advice.py index 11008b2f375345a98cd1d20d22333a2138c8dfa9..ac163a794863b3837deb006cf26dbc27579e3b74 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/op_schedule_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/op_schedule_advice.py @@ -15,8 +15,8 @@ import logging from decimal import Decimal -from common_func_advisor.constant import Constant -from timeline_advice.timeline_advice_base import TimelineAdviceBase +from msprof_analyze.advisor.advisor_backend.common_func_advisor.constant import Constant +from msprof_analyze.advisor.advisor_backend.timeline_advice.timeline_advice_base import TimelineAdviceBase logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/optimizer_advice.py b/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/optimizer_advice.py index 6d6bf7e9b96ef09e755021bf08a634217f6105d9..6a01b1694eab10206eed3cd0e0e36f1dba8a61aa 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/optimizer_advice.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/optimizer_advice.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from timeline_advice.timeline_advice_base import TimelineAdviceBase +from msprof_analyze.advisor.advisor_backend.timeline_advice.timeline_advice_base import TimelineAdviceBase class OptimizerAdvice(TimelineAdviceBase): diff --git a/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/timeline_advice_base.py b/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/timeline_advice_base.py index e712799a96bde4537c5cb1af4c254d42c3c3e585..36c5e1e2dea6dcd41f3199e1c542ae4eebde1663 100644 --- a/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/timeline_advice_base.py +++ b/profiler/msprof_analyze/advisor/advisor_backend/timeline_advice/timeline_advice_base.py @@ -15,12 +15,10 @@ import logging import os -import json from abc import abstractmethod from collections import defaultdict -from advice_base import AdviceBase - +from msprof_analyze.advisor.advisor_backend.advice_base import AdviceBase from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/analyzer/analyzer_controller.py b/profiler/msprof_analyze/advisor/analyzer/analyzer_controller.py index fde739eeb14d19b17982df180b5f492a2b203a34..bde9e5cd3454a85853a6fbcfbd0ade060ebc229b 100644 --- a/profiler/msprof_analyze/advisor/analyzer/analyzer_controller.py +++ b/profiler/msprof_analyze/advisor/analyzer/analyzer_controller.py @@ -24,9 +24,6 @@ from pathlib import Path import psutil -sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "compare_tools")) -sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "cluster_analyse")) - from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager from msprof_analyze.advisor.analyzer.cluster.slow_rank_analyzer import SlowRankAnalyzer from msprof_analyze.advisor.analyzer.cluster.slow_link_analyzer import SlowLinkAnalyzer diff --git a/profiler/msprof_analyze/advisor/analyzer/cluster/slow_link_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/cluster/slow_link_analyzer.py index 062b6548e1a91db03bb1c8c58f3261b020b577e1..9c4416009e1035e1938cf5430a49b0383bbf47d7 100644 --- a/profiler/msprof_analyze/advisor/analyzer/cluster/slow_link_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/cluster/slow_link_analyzer.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import defaultdict from typing import Dict, List import logging diff --git a/profiler/msprof_analyze/advisor/analyzer/computation/pp_stage_computation_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/computation/pp_stage_computation_analyzer.py index 538562cb1fb33f09099fa7b839e331f6d2430fe8..2780204b2064ed628ee686d91e82169818955eb7 100644 --- a/profiler/msprof_analyze/advisor/analyzer/computation/pp_stage_computation_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/computation/pp_stage_computation_analyzer.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -import os from multiprocessing import Manager from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer diff --git a/profiler/msprof_analyze/advisor/analyzer/graph_fusion/graph_fusion_checker.py b/profiler/msprof_analyze/advisor/analyzer/graph_fusion/graph_fusion_checker.py index 1e2a66695d2c7e1d2b271ef4e515ed3d129e9e56..727c5a3770782332a611321e7e566f14fbe8a3ee 100644 --- a/profiler/msprof_analyze/advisor/analyzer/graph_fusion/graph_fusion_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/graph_fusion/graph_fusion_checker.py @@ -24,7 +24,6 @@ from msprof_analyze.advisor.common.graph.graph import Graph from msprof_analyze.advisor.common.graph.graph_parser import QueryGraphParser from msprof_analyze.advisor.dataset.graph_dataset import GraphDataset from msprof_analyze.advisor.common.graph.graph_match import find_isomorphisms -from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/analyzer/memory/memory_checker.py b/profiler/msprof_analyze/advisor/analyzer/memory/memory_checker.py index 7288bf8a5ec2863c75ee561458d44728be26480e..82bca84cd233aadf0e0744d2dab51c341a19e3cf 100644 --- a/profiler/msprof_analyze/advisor/analyzer/memory/memory_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/memory/memory_checker.py @@ -12,10 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os import re import logging -import yaml from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset, MemCollector from msprof_analyze.advisor.result.result import OptimizeResult diff --git a/profiler/msprof_analyze/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py b/profiler/msprof_analyze/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py index 1c5c325be13bab411262e17d4aa0e173ed833dea..247088080b9a5e1b492889752405516a1a731f3e 100644 --- a/profiler/msprof_analyze/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py @@ -22,7 +22,6 @@ from tqdm import tqdm from msprof_analyze.advisor.analyzer.base_analyzer import BaseAnalyzer from msprof_analyze.advisor.display.prompt.base_prompt import BasePrompt from msprof_analyze.prof_common.constant import Constant -from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes from msprof_analyze.advisor.common.timeline.event import TimelineEvent from msprof_analyze.advisor.config.config import Config from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset @@ -30,7 +29,6 @@ from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord from msprof_analyze.advisor.utils.utils import format_timeline_result from msprof_analyze.advisor.common.timeline.fusion_ops_db import init_timeline_ops_db from msprof_analyze.advisor.display.html.priority_background_color import PriorityBackgroundColor -from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/analyzer/schedule/gc/gc_checker.py b/profiler/msprof_analyze/advisor/analyzer/schedule/gc/gc_checker.py index 92a0d208f8118f09ad6aae6a2f5899d8a7c842ca..8e9f9139afd4eac8a80e6f27b34b399cebb1e65e 100644 --- a/profiler/msprof_analyze/advisor/analyzer/schedule/gc/gc_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/gc/gc_checker.py @@ -21,7 +21,6 @@ from msprof_analyze.advisor.result.result import OptimizeResult from msprof_analyze.advisor.result.item import OptimizeItem, OptimizeRecord from msprof_analyze.advisor.utils.utils import convert_to_float, convert_to_int from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager -from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import FileManager logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/analyzer/schedule/timeline_base_checker.py b/profiler/msprof_analyze/advisor/analyzer/schedule/timeline_base_checker.py index 99ca99e99eb92aab3e89ae48383bdf0ad7d1c9dc..649e41e5daa7597795d31abb5ed6468364cf49ff 100644 --- a/profiler/msprof_analyze/advisor/analyzer/schedule/timeline_base_checker.py +++ b/profiler/msprof_analyze/advisor/analyzer/schedule/timeline_base_checker.py @@ -12,14 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from abc import ABC, abstractmethod +from abc import ABC import multiprocessing import logging from msprof_analyze.prof_common.constant import Constant from msprof_analyze.advisor.common.timeline.event import TimelineEvent from msprof_analyze.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset -from msprof_analyze.advisor.result.result import OptimizeResult logger = logging.getLogger() diff --git a/profiler/msprof_analyze/advisor/cluster_perf_analysis.ipynb b/profiler/msprof_analyze/advisor/cluster_perf_analysis.ipynb index e3b6b1afd76fc7edc6a9e9c0a375dcf4e071cd04..0fc37222f6829331f30ba6dd470faae475a1bac9 100644 --- a/profiler/msprof_analyze/advisor/cluster_perf_analysis.ipynb +++ b/profiler/msprof_analyze/advisor/cluster_perf_analysis.ipynb @@ -675,7 +675,7 @@ "metadata": {}, "outputs": [], "source": [ - "from advisor_backend.interface import Interface\n", + "from msprof_analyze.advisor.advisor_backend.interface import Interface\n", "import matplotlib.pyplot as plt\n", "import numpy as np" ] diff --git a/profiler/msprof_analyze/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py b/profiler/msprof_analyze/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py index a7490c2e51220b80a7c30768f51e327bc56b84c6..71b3e7410727f5c74f6e502c8c76787fcf3a4a0d 100644 --- a/profiler/msprof_analyze/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py +++ b/profiler/msprof_analyze/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py @@ -16,11 +16,6 @@ import json import logging import math -import os -import traceback - -import ijson -from tqdm import tqdm from msprof_analyze.advisor.common.timeline.event import TimelineEvent from msprof_analyze.advisor.utils.utils import get_file_path_from_directory diff --git a/profiler/msprof_analyze/advisor/dataset/timeline_event_dataset.py b/profiler/msprof_analyze/advisor/dataset/timeline_event_dataset.py index e1eb98171a476602a8720e85706a95a457b39f42..418883f6ce94ce18a9dacc408c2e1b707f1970a1 100644 --- a/profiler/msprof_analyze/advisor/dataset/timeline_event_dataset.py +++ b/profiler/msprof_analyze/advisor/dataset/timeline_event_dataset.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import inspect import logging import traceback from collections import OrderedDict diff --git a/profiler/msprof_analyze/advisor/display/html/render.py b/profiler/msprof_analyze/advisor/display/html/render.py index ffc07509ada5a1d58fa059387cbb9e5088134990..09211220677a0d7d50e8af52a27f068a85d0859b 100644 --- a/profiler/msprof_analyze/advisor/display/html/render.py +++ b/profiler/msprof_analyze/advisor/display/html/render.py @@ -20,7 +20,6 @@ from collections import defaultdict, OrderedDict from jinja2 import Environment, FileSystemLoader from msprof_analyze.prof_common.constant import Constant - from msprof_analyze.advisor.config.config import Config from msprof_analyze.advisor.utils.utils import singleton, safe_write diff --git a/profiler/msprof_analyze/advisor/interface/interface.py b/profiler/msprof_analyze/advisor/interface/interface.py index e69dc41e3a3589d1ded3d9b57061084159cbf238..b3afefee57c8c62030af17130f79413238588f8f 100644 --- a/profiler/msprof_analyze/advisor/interface/interface.py +++ b/profiler/msprof_analyze/advisor/interface/interface.py @@ -14,14 +14,8 @@ # limitations under the License. import os from collections import OrderedDict -import sys import logging -sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), - "cluster_analyse")) -sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), - "compare_tools")) - from msprof_analyze.advisor.utils.utils import Timer from msprof_analyze.advisor.result.result import OptimizeResult from msprof_analyze.advisor.analyzer.computation.profiling_analyzer import AicpuAnalyzer, BlockDimAnalyzer, \ diff --git a/profiler/msprof_analyze/advisor/result/result.py b/profiler/msprof_analyze/advisor/result/result.py index 2953ca5dbe9dcd36bbb4553ff93fee64377ec218..422fd43ec4a3c5652aa5a3a1271cbba4c8bdd50e 100644 --- a/profiler/msprof_analyze/advisor/result/result.py +++ b/profiler/msprof_analyze/advisor/result/result.py @@ -15,7 +15,6 @@ import json import os -import stat from collections import OrderedDict import click @@ -26,7 +25,7 @@ from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsMan from msprof_analyze.prof_common.constant import Constant from msprof_analyze.advisor.utils.utils import singleton, logger from msprof_analyze.advisor.config.config import Config -from msprof_analyze.advisor.utils.file import FdOpen, check_dir_writable +from msprof_analyze.advisor.utils.file import check_dir_writable from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/advisor/utils/utils.py b/profiler/msprof_analyze/advisor/utils/utils.py index 03d098fc371a2101ab1baf9939754934ca7b4a3b..2ddf91c76a3052fffd8fa3cd8f5a92b6b6f52ea9 100644 --- a/profiler/msprof_analyze/advisor/utils/utils.py +++ b/profiler/msprof_analyze/advisor/utils/utils.py @@ -29,12 +29,9 @@ from functools import wraps from typing import Any, Set import ijson import click -import requests -from requests.adapters import HTTPAdapter from tqdm import tqdm from msprof_analyze.prof_common.constant import Constant -from msprof_analyze.advisor.common.version_control import VersionControl from msprof_analyze.advisor.utils.log import init_logger, get_log_level logger = logging.getLogger() diff --git a/profiler/msprof_analyze/cli/analyze_cli.py b/profiler/msprof_analyze/cli/analyze_cli.py index 73cbf6b7490ecf361ba08b3926f83167130edd87..69c77e04e021bc13fbaea5634cc09e1ccf8c9092 100644 --- a/profiler/msprof_analyze/cli/analyze_cli.py +++ b/profiler/msprof_analyze/cli/analyze_cli.py @@ -13,14 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys -import os import logging import click -sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__)), "compare_tools")) -sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__)), "cluster_analyse")) - from msprof_analyze.prof_common.path_manager import PathManager from msprof_analyze.advisor.analyzer.analyzer_controller import AnalyzerController from msprof_analyze.advisor.utils.tools import CONTEXT_SETTINGS, ClickAliasedGroup diff --git a/profiler/msprof_analyze/cli/cluster_cli.py b/profiler/msprof_analyze/cli/cluster_cli.py index 12006425548e763649caf0a78d04b4ee76156cd9..0cdb2bd2b10b2ede411d10221e36a51e3f015e12 100644 --- a/profiler/msprof_analyze/cli/cluster_cli.py +++ b/profiler/msprof_analyze/cli/cluster_cli.py @@ -12,12 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os -import sys import click -sys.path.append(os.path.dirname(os.path.dirname(__file__))) - from msprof_analyze.prof_common.constant import Constant from msprof_analyze.cluster_analyse.cluster_analysis import ALL_FEATURE_LIST, Interface from msprof_analyze.prof_common.path_manager import PathManager diff --git a/profiler/msprof_analyze/cli/compare_cli.py b/profiler/msprof_analyze/cli/compare_cli.py index c7af038d0cff677e29d0c349b8c06c3fd8954b36..7ef8139ae0133ed0de155609f94068d1d6ab42d9 100644 --- a/profiler/msprof_analyze/cli/compare_cli.py +++ b/profiler/msprof_analyze/cli/compare_cli.py @@ -13,12 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import ast -import os -import sys import click -sys.path.append(os.path.dirname(os.path.dirname(__file__))) - from msprof_analyze.prof_common.path_manager import PathManager from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.analyze_dict import AnalyzeDict diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/analysis_facade.py b/profiler/msprof_analyze/cluster_analyse/analysis/analysis_facade.py index 74c53e769febdc5efca47d9c88377a258589fed3..aa9658f0c6229264e40eeacf6cfcc9fcd3a18538 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/analysis_facade.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/analysis_facade.py @@ -12,18 +12,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import threading from multiprocessing import Process, Value, Lock from tqdm import tqdm -from analysis.communication_analysis import CommunicationAnalysis -from analysis.communication_analysis import CommunicationAnalysisOptimized -from analysis.comm_matrix_analysis import CommMatrixAnalysis -from analysis.comm_matrix_analysis import CommMatrixAnalysisOptimized -from analysis.step_trace_time_analysis import StepTraceTimeAnalysis -from analysis.host_info_analysis import HostInfoAnalysis -from analysis.cluster_base_info_analysis import ClusterBaseInfoAnalysis -from common_func.context import Context +from msprof_analyze.cluster_analyse.analysis.communication_analysis import CommunicationAnalysis +from msprof_analyze.cluster_analyse.analysis.communication_analysis import CommunicationAnalysisOptimized +from msprof_analyze.cluster_analyse.analysis.comm_matrix_analysis import CommMatrixAnalysis +from msprof_analyze.cluster_analyse.analysis.comm_matrix_analysis import CommMatrixAnalysisOptimized +from msprof_analyze.cluster_analyse.analysis.step_trace_time_analysis import StepTraceTimeAnalysis +from msprof_analyze.cluster_analyse.analysis.host_info_analysis import HostInfoAnalysis +from msprof_analyze.cluster_analyse.analysis.cluster_base_info_analysis import ClusterBaseInfoAnalysis +from msprof_analyze.cluster_analyse.common_func.context import Context from msprof_analyze.cluster_analyse.common_func.analysis_loader import get_class_from_name from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/base_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/base_analysis.py index 145cefa66caf1eb17e3773bccada24516bc67799..59b824c1defd3ea93655a71d9727962871d07537 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/base_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/base_analysis.py @@ -14,8 +14,7 @@ # limitations under the License. from abc import abstractmethod -from cluster_utils.data_transfer_adapter import DataTransferAdapter - +from msprof_analyze.cluster_analyse.cluster_utils.data_transfer_adapter import DataTransferAdapter from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import FileManager from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/cluster_base_info_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/cluster_base_info_analysis.py index 9e89cdc4209e3e0cb14b6eb1725bb4545a030711..bb0d73041b6ded5564a9510c17abd594841eaf23 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/cluster_base_info_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/cluster_base_info_analysis.py @@ -14,13 +14,11 @@ # limitations under the License. import json import os -import logging - -from analysis.base_analysis import BaseAnalysis -from common_func.db_manager import DBManager -from common_func.utils import increase_shared_value -from common_func.path_manager import PathManager +from msprof_analyze.cluster_analyse.analysis.base_analysis import BaseAnalysis +from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value +from msprof_analyze.cluster_analyse.common_func.path_manager import PathManager from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py index b3e3c8ddae6850ad92036a69563bfb84ccac1576..5ce5b1387a2c7e3fad2c4270526d93e46105c6bb 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py @@ -16,10 +16,9 @@ import os from collections import defaultdict -from analysis.base_analysis import BaseAnalysis -from common_func.db_manager import DBManager -from common_func.utils import increase_shared_value - +from msprof_analyze.cluster_analyse.analysis.base_analysis import BaseAnalysis +from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py index 1f7910b81717e4f5f229dcc1d0a469ab4353a0b9..2a9e01fc1371b75b9b5717a266fd3b22a4d90b6b 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/communication_analysis.py @@ -16,12 +16,12 @@ import os from collections import defaultdict -from analysis.base_analysis import BaseAnalysis -from common_func.table_constant import TableConstant -from common_func.db_manager import DBManager -from common_func.utils import increase_shared_value -from prof_bean.communication_bandwidth_bean import CommunicationBandwidthBean -from prof_bean.communication_time_bean import CommunicationTimeBean +from msprof_analyze.cluster_analyse.analysis.base_analysis import BaseAnalysis +from msprof_analyze.cluster_analyse.common_func.table_constant import TableConstant +from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value +from msprof_analyze.cluster_analyse.prof_bean.communication_bandwidth_bean import CommunicationBandwidthBean +from msprof_analyze.cluster_analyse.prof_bean.communication_time_bean import CommunicationTimeBean from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py index abef72d5ffb7758f6df5432c91333c1f6927a4a6..dc43e313c5baab8f57db7fca03b2130c33a73724 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/host_info_analysis.py @@ -15,11 +15,10 @@ import os -from analysis.base_analysis import BaseAnalysis -from common_func.db_manager import DBManager -from common_func.utils import increase_shared_value -from common_func.path_manager import PathManager - +from msprof_analyze.cluster_analyse.analysis.base_analysis import BaseAnalysis +from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value +from msprof_analyze.cluster_analyse.common_func.path_manager import PathManager from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py index 1fbc8b8dad4197ea5a1cb0e3393fdac5f4ba25b9..a7cffdd176e6ad0c51a9922a9853e1b20f260322 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/step_trace_time_analysis.py @@ -14,11 +14,10 @@ # limitations under the License. import os -from common_func.db_manager import DBManager -from common_func.utils import increase_shared_value -from cluster_utils.parallel_strategy_calculator import ParallelStrategyCalculator -from prof_bean.step_trace_time_bean import StepTraceTimeBean - +from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value +from msprof_analyze.cluster_analyse.cluster_utils.parallel_strategy_calculator import ParallelStrategyCalculator +from msprof_analyze.cluster_analyse.prof_bean.step_trace_time_bean import StepTraceTimeBean from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import FileManager from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/cluster_analysis.py b/profiler/msprof_analyze/cluster_analyse/cluster_analysis.py index 4cb486608f0007431e6f683754b5452ede3e54cf..6464bb732ddf57b2790d99ac7148ce3ecaf327ce 100644 --- a/profiler/msprof_analyze/cluster_analyse/cluster_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/cluster_analysis.py @@ -18,10 +18,10 @@ import sys sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) -from analysis.analysis_facade import AnalysisFacade -from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor -from cluster_data_preprocess.mindspore_data_preprocessor import MindsporeDataPreprocessor -from communication_group.communication_group_generator import CommunicationGroupGenerator +from msprof_analyze.cluster_analyse.analysis.analysis_facade import AnalysisFacade +from msprof_analyze.cluster_analyse.cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor +from msprof_analyze.cluster_analyse.cluster_data_preprocess.mindspore_data_preprocessor import MindsporeDataPreprocessor +from msprof_analyze.cluster_analyse.communication_group.communication_group_generator import CommunicationGroupGenerator from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/mindspore_data_preprocessor.py b/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/mindspore_data_preprocessor.py index 3a1a91230d57bd8e5f3800de98807011629ddb95..eaa14fb71f9583b65b0ed18c6ad9727d913eb2fe 100644 --- a/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/mindspore_data_preprocessor.py +++ b/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/mindspore_data_preprocessor.py @@ -14,7 +14,7 @@ # limitations under the License. from collections import defaultdict -from cluster_data_preprocess.data_preprocessor import DataPreprocessor +from msprof_analyze.cluster_analyse.cluster_data_preprocess.data_preprocessor import DataPreprocessor from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py b/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py index fb2be0860d9976cad04665cc58511d52d3f61b9e..d27060590624f57a8743a5117b73528dadefd5ae 100644 --- a/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py +++ b/profiler/msprof_analyze/cluster_analyse/cluster_data_preprocess/pytorch_data_preprocessor.py @@ -15,7 +15,7 @@ from collections import defaultdict import os -from cluster_data_preprocess.data_preprocessor import DataPreprocessor +from msprof_analyze.cluster_analyse.cluster_data_preprocess.data_preprocessor import DataPreprocessor from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import FileManager from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/cluster_utils/data_transfer_adapter.py b/profiler/msprof_analyze/cluster_analyse/cluster_utils/data_transfer_adapter.py index a9ee57e57ca906c7b6da82d48fbd7db9923c5b36..b45b5b3ffb0b6493100bdf5b3f0dc3b9cce561c4 100644 --- a/profiler/msprof_analyze/cluster_analyse/cluster_utils/data_transfer_adapter.py +++ b/profiler/msprof_analyze/cluster_analyse/cluster_utils/data_transfer_adapter.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import copy -from common_func.table_constant import TableConstant +from msprof_analyze.cluster_analyse.common_func.table_constant import TableConstant from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/cluster_analyse/cluster_utils/parallel_strategy_calculator.py b/profiler/msprof_analyze/cluster_analyse/cluster_utils/parallel_strategy_calculator.py index 0f0a1809d99e5a6226e7d88955286eed8bf4132c..d6a8dd4057103f380be855d642dc74fa1cac1972 100644 --- a/profiler/msprof_analyze/cluster_analyse/cluster_utils/parallel_strategy_calculator.py +++ b/profiler/msprof_analyze/cluster_analyse/cluster_utils/parallel_strategy_calculator.py @@ -15,7 +15,7 @@ from enum import Enum from dataclasses import dataclass -from .parallel_algorithm import MegatronAlgorithm +from msprof_analyze.cluster_analyse.cluster_utils.parallel_algorithm import MegatronAlgorithm class ParallelAlgorithmType(Enum): diff --git a/profiler/msprof_analyze/cluster_analyse/common_func/db_manager.py b/profiler/msprof_analyze/cluster_analyse/common_func/db_manager.py index 0abb2bce190e324c38bb3c2ab0460489d50e3074..fb78057064426869cde23e1ff4e0755e709f9ee9 100644 --- a/profiler/msprof_analyze/cluster_analyse/common_func/db_manager.py +++ b/profiler/msprof_analyze/cluster_analyse/common_func/db_manager.py @@ -16,9 +16,9 @@ import os import sqlite3 -from common_func.empty_class import EmptyClass -from common_func.tables_config import TablesConfig -from common_func.sql_extention_func import SqlExtentionAggregateFunc +from msprof_analyze.cluster_analyse.common_func.empty_class import EmptyClass +from msprof_analyze.cluster_analyse.common_func.tables_config import TablesConfig +from msprof_analyze.cluster_analyse.common_func.sql_extention_func import SqlExtentionAggregateFunc from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import check_db_path_valid diff --git a/profiler/msprof_analyze/cluster_analyse/communication_group/base_communication_group.py b/profiler/msprof_analyze/cluster_analyse/communication_group/base_communication_group.py index c15662ac0fbb7c4c9dd3a746d344ff629c82cd17..8f6625f8f6bbf646cfd77099b70c36398680f67a 100644 --- a/profiler/msprof_analyze/cluster_analyse/communication_group/base_communication_group.py +++ b/profiler/msprof_analyze/cluster_analyse/communication_group/base_communication_group.py @@ -19,7 +19,7 @@ from collections import defaultdict from copy import deepcopy from multiprocessing import Pool -from cluster_utils.data_transfer_adapter import DataTransferAdapter +from msprof_analyze.cluster_analyse.cluster_utils.data_transfer_adapter import DataTransferAdapter from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py index a3b1ef372539fa60baf195e749f86d4d09a28a5b..55f0b7802c61e02ad7900083102fd74ff0a0d411 100644 --- a/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py +++ b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_db_group.py @@ -14,8 +14,8 @@ # limitations under the License. import os -from common_func.db_manager import DBManager -from communication_group.base_communication_group import BaseCommunicationGroup +from msprof_analyze.cluster_analyse.common_func.db_manager import DBManager +from msprof_analyze.cluster_analyse.communication_group.base_communication_group import BaseCommunicationGroup from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/communication_group/communication_group_generator.py b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_group_generator.py index 00ced8e8bd86fc0eb5e06cd182fd61dde9c62f3b..39a51e5070955aa46fd16411b1a020136ced6a21 100644 --- a/profiler/msprof_analyze/cluster_analyse/communication_group/communication_group_generator.py +++ b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_group_generator.py @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from communication_group.communication_db_group import CommunicationDBGroup -from communication_group.communication_db_group import CommunicationDBGroupOptimized -from communication_group.communication_json_group import CommunicationJsonGroup +from msprof_analyze.cluster_analyse.communication_group.communication_db_group import CommunicationDBGroup +from msprof_analyze.cluster_analyse.communication_group.communication_db_group import CommunicationDBGroupOptimized +from msprof_analyze.cluster_analyse.communication_group.communication_json_group import CommunicationJsonGroup from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/cluster_analyse/communication_group/communication_json_group.py b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_json_group.py index 2d39c8ef7ddaf025a44986e10c783ecb06a91144..97948228264f7b6fb2aed8d8b8766b3515626d40 100644 --- a/profiler/msprof_analyze/cluster_analyse/communication_group/communication_json_group.py +++ b/profiler/msprof_analyze/cluster_analyse/communication_group/communication_json_group.py @@ -15,7 +15,7 @@ import os -from communication_group.base_communication_group import BaseCommunicationGroup +from msprof_analyze.cluster_analyse.communication_group.base_communication_group import BaseCommunicationGroup from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_bandwidth_bean.py b/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_bandwidth_bean.py index 3c3190ea414cc2279caee6f455b460d688d72e0f..e3ae1455419667f3d87b6520cc01ee718d48dc87 100644 --- a/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_bandwidth_bean.py +++ b/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_bandwidth_bean.py @@ -14,7 +14,7 @@ # limitations under the License. -from common_func.table_constant import TableConstant +from msprof_analyze.cluster_analyse.common_func.table_constant import TableConstant class CommunicationBandwidthBean: diff --git a/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_time_bean.py b/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_time_bean.py index 5bc4603d290a9fe05e538f2fba9fa88eab949dbc..704e9416aa54ef83bade99e1c6897fa0395a127b 100644 --- a/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_time_bean.py +++ b/profiler/msprof_analyze/cluster_analyse/prof_bean/communication_time_bean.py @@ -14,7 +14,7 @@ # limitations under the License. -from common_func.table_constant import TableConstant +from msprof_analyze.cluster_analyse.common_func.table_constant import TableConstant class CommunicationTimeBean: diff --git a/profiler/msprof_analyze/cluster_analyse/recipes/cann_api_sum/cann_api_sum.py b/profiler/msprof_analyze/cluster_analyse/recipes/cann_api_sum/cann_api_sum.py index 3df4f143f1b66f7fedf52c9ecaccd338d2a834e5..17f8f698960ac4ebd10bfbdfd0c5712fa016b7f1 100644 --- a/profiler/msprof_analyze/cluster_analyse/recipes/cann_api_sum/cann_api_sum.py +++ b/profiler/msprof_analyze/cluster_analyse/recipes/cann_api_sum/cann_api_sum.py @@ -16,7 +16,7 @@ import os import pandas as pd -from common_func.utils import stdev +from msprof_analyze.cluster_analyse.common_func.utils import stdev from msprof_analyze.cluster_analyse.recipes.base_recipe_analysis import BaseRecipeAnalysis from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/recipes/compute_op_sum/compute_op_sum.py b/profiler/msprof_analyze/cluster_analyse/recipes/compute_op_sum/compute_op_sum.py index 9706efb656d6ff4b3b15ae0caab1973ebd6653ef..a5d44c3f17f6d2a31f097506c38d829c18d5d74f 100644 --- a/profiler/msprof_analyze/cluster_analyse/recipes/compute_op_sum/compute_op_sum.py +++ b/profiler/msprof_analyze/cluster_analyse/recipes/compute_op_sum/compute_op_sum.py @@ -16,7 +16,7 @@ import os import pandas as pd -from common_func.utils import describe_duration +from msprof_analyze.cluster_analyse.common_func.utils import describe_duration from msprof_analyze.cluster_analyse.recipes.base_recipe_analysis import BaseRecipeAnalysis from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/recipes/hccl_sum/hccl_sum.py b/profiler/msprof_analyze/cluster_analyse/recipes/hccl_sum/hccl_sum.py index 83087ae98e5a9c1259dc719499a30a693b1b6513..a78603ee0ac2894fb8b60a21f411e7fef9d144db 100644 --- a/profiler/msprof_analyze/cluster_analyse/recipes/hccl_sum/hccl_sum.py +++ b/profiler/msprof_analyze/cluster_analyse/recipes/hccl_sum/hccl_sum.py @@ -16,8 +16,7 @@ import os import pandas as pd -from common_func.utils import describe_duration - +from msprof_analyze.cluster_analyse.common_func.utils import describe_duration from msprof_analyze.cluster_analyse.recipes.base_recipe_analysis import BaseRecipeAnalysis from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/cluster_analyse/recipes/mstx_sum/mstx_sum.py b/profiler/msprof_analyze/cluster_analyse/recipes/mstx_sum/mstx_sum.py index 829c2dda8b732e626fda8645ebae9c0c33c3db66..69b4b056850b85a856634c7feb0121cfcb34494b 100644 --- a/profiler/msprof_analyze/cluster_analyse/recipes/mstx_sum/mstx_sum.py +++ b/profiler/msprof_analyze/cluster_analyse/recipes/mstx_sum/mstx_sum.py @@ -17,7 +17,7 @@ from collections import namedtuple import os import pandas as pd -from common_func.utils import describe_duration +from msprof_analyze.cluster_analyse.common_func.utils import describe_duration from msprof_analyze.cluster_analyse.recipes.base_recipe_analysis import BaseRecipeAnalysis from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/api_compare_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/api_compare_comparator.py index 6dd4ed5fc7b39e118edd7fe3537446a450976948..bea449255768f7d5c5ed597682e47fafad2f2802 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/api_compare_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/api_compare_comparator.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.utils.common_func import update_order_id - +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.utils.common_func import update_order_id from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/communication_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/communication_comparator.py index a9aa221d3332c788e1edca87bb93a9d3823a7c6e..7710d2cbb7fd1bc2c67088d59dfd6f9ad618fa7a 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/communication_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/communication_comparator.py @@ -12,9 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.compare_bean.communication_bean import CommunicationBean -from compare_backend.utils.common_func import update_order_id +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.communication_bean import CommunicationBean +from msprof_analyze.compare_tools.compare_backend.utils.common_func import update_order_id from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_compare_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_compare_comparator.py index 51fce7c1568b23d01e863eaadb99b29f4a1dd728..89e1d3e0ada4982a8c32bd0325ec0e9cad537746 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_compare_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_compare_comparator.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.utils.common_func import update_order_id - +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.utils.common_func import update_order_id from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_type_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_type_comparator.py index ebca4346fab2e4df7454e3c1cb219a89c56b4a7e..44e117c82f335a04d05c6c01534b57c5f6c33d01 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_type_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/kernel_type_comparator.py @@ -12,10 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.compare_bean.origin_data_bean.op_stastic_bean import OpStatisticBean -from compare_backend.utils.common_func import update_order_id - +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.op_stastic_bean import OpStatisticBean +from msprof_analyze.compare_tools.compare_backend.utils.common_func import update_order_id from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_comparetor.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_comparetor.py index 3b13c244114ab9c4f0439c933a97979459d69ca1..11c701a55a223b5e42404e9841edb284c2bc3720 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_comparetor.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_comparetor.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.utils.common_func import update_order_id - +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.utils.common_func import update_order_id from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_statistic_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_statistic_comparator.py index a31d1f32a506bae44a0d4c9b8adf9805539ce09f..7fee4a361dfefce9ab351b1061ffdeddb831e5e9 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_statistic_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/module_statistic_comparator.py @@ -14,8 +14,8 @@ # limitations under the License. from collections import OrderedDict -from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.utils.common_func import update_order_id +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.utils.common_func import update_order_id class ModuleStatisticComparator(BaseComparator): diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_comparator.py index b6476d422ecc8472e1c40dd4560696c4e1dbcc67..1091da79c34bcbcf23cc7c4d4d38729362106a72 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_comparator.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator class OperatorComparator(BaseComparator): diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_statistic_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_statistic_comparator.py index 56610e9734cc333464906dd8e7296e06e4a53154..db1db36f724aab6f84165459090b99d6be5c6d6a 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_statistic_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/operator_statistic_comparator.py @@ -12,8 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.utils.common_func import update_order_id +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.utils.common_func import update_order_id class OperatorStatisticComparator(BaseComparator): diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_metrics_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_metrics_comparator.py index ad853f2db803b3e476a0af668ca4db60cf32283a..8a80262c6abc91e4db399f993a502faff9fb4ace 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_metrics_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_metrics_comparator.py @@ -14,8 +14,8 @@ # limitations under the License. from math import isclose -from compare_backend.comparator.base_comparator import BaseComparator -from compare_backend.utils.excel_config import ExcelConfig +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_performance_comparator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_performance_comparator.py index 6adf9e0d6f7639df672b1bd11b7b61ca2f2f9652..b1c11a71e726d83f1f828b9bd0c9c09226cec5f7 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_performance_comparator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparator/overall_performance_comparator.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.base_comparator import BaseComparator +from msprof_analyze.compare_tools.compare_backend.comparator.base_comparator import BaseComparator from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/api_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/api_compare_bean.py index 5178485700ca30d3b25f9a4d5d057c62eb08dfcb..faf98fb96d36bc08a38926dd0207b72a5f7b2c56 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/api_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/api_compare_bean.py @@ -12,8 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.excel_config import ExcelConfig +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/communication_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/communication_bean.py index f840eb653c032a222ca56cac182c0d1da982f277..1801a75cb807959fa26addbee8a563be6ebab434 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/communication_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/communication_bean.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.excel_config import ExcelConfig -from compare_backend.utils.common_func import calculate_diff_ratio - +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py index ff10d94dd5bf4238a531949a3b746a42df571891..3f64f28ec6c62b2c61548b7d2f189339670aea06 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import calculate_diff_ratio, convert_to_float -from compare_backend.utils.excel_config import ExcelConfig - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio, convert_to_float +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_type_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_type_compare_bean.py index b129facc946a5aca9824fd3bf05ae3592f869d0f..1434f42d5f6a0d0300010700cf638536c6b4b03e 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_type_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/kernel_type_compare_bean.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.excel_config import ExcelConfig - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_compare_bean.py index 5e2720a2bb8588791bc2e04f9b98a61dd7ef841b..7a269d6dbcca4393fcd937d3cc2ab69905f6bb2b 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_compare_bean.py @@ -12,11 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.excel_config import ExcelConfig -from compare_backend.utils.torch_op_node import TorchOpNode -from compare_backend.utils.tree_builder import TreeBuilder - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig +from msprof_analyze.compare_tools.compare_backend.utils.torch_op_node import TorchOpNode +from msprof_analyze.compare_tools.compare_backend.utils.tree_builder import TreeBuilder from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py index 18ca502e72c431ffe1f94fc61d0804b51e3c85df..e4f7792b5eb3e12aa0c66f784cc827b7761bf901 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/memory_statistic_bean.py @@ -12,10 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.tree_builder import TreeBuilder -from compare_backend.utils.excel_config import ExcelConfig - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.tree_builder import TreeBuilder +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_compare_bean.py index 9dee49acda7eca18ddeb404577dfa50313068ff8..107a2d79caae5c21cc6ef71761ba2447cc58cc11 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_compare_bean.py @@ -12,12 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import longest_common_subsequence_matching, calculate_diff_ratio -from compare_backend.utils.excel_config import ExcelConfig -from compare_backend.utils.module_node import ModuleNode -from compare_backend.utils.name_function import NameFunction -from compare_backend.utils.torch_op_node import TorchOpNode - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import ( + longest_common_subsequence_matching, + calculate_diff_ratio +) +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig +from msprof_analyze.compare_tools.compare_backend.utils.module_node import ModuleNode +from msprof_analyze.compare_tools.compare_backend.utils.name_function import NameFunction +from msprof_analyze.compare_tools.compare_backend.utils.torch_op_node import TorchOpNode from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_statistic_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_statistic_bean.py index b4448d1eb9975cbc41601effa122df7d9a216cf0..2ca8b7e11946566038747cd8916d2a843daa4601 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_statistic_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/module_statistic_bean.py @@ -14,9 +14,8 @@ # limitations under the License. import re -from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.excel_config import ExcelConfig - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_compare_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_compare_bean.py index b475a59ef5e3b764d282e37ec9f70a03e3d075af..3770afc8445ee5234c59682b21a2a19c3186e379 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_compare_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_compare_bean.py @@ -12,11 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.excel_config import ExcelConfig -from compare_backend.utils.torch_op_node import TorchOpNode -from compare_backend.utils.tree_builder import TreeBuilder - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig +from msprof_analyze.compare_tools.compare_backend.utils.torch_op_node import TorchOpNode +from msprof_analyze.compare_tools.compare_backend.utils.tree_builder import TreeBuilder from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py index 11d555576c776be7ebd22b300b9c49e88224db42..ab13584cb5e2ed59497a52bb8a387f6a5bab440b 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/operator_statistic_bean.py @@ -12,10 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.excel_config import ExcelConfig -from compare_backend.utils.tree_builder import TreeBuilder - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig +from msprof_analyze.compare_tools.compare_backend.utils.tree_builder import TreeBuilder from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py index 8c9d84075f2700ff491c5558b3aee1d09b236d5e..1f921334aa3615d838a1086302c5b52c17a0ec34 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/compare_event.py @@ -14,8 +14,7 @@ # limitations under the License. from decimal import Decimal -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean - +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py index f1ff9f575adaec9708d2a615bec1d86db0c0defd..d9f6e519da264836673ae5a58fca64f3acd06109 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py @@ -17,9 +17,8 @@ from decimal import Decimal import pandas as pd -from compare_backend.utils.common_func import convert_to_float, convert_to_decimal -from compare_backend.compare_config.compare_config import CompareConfig - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import convert_to_float, convert_to_decimal +from msprof_analyze.compare_tools.compare_backend.compare_config.compare_config import CompareConfig from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/memory_record_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/memory_record_bean.py index 6d834d724de4c46df0df6fc6dda78a950eaa3595..e123d53fab53a5d4c371cb39026a8e2818ba32df 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/memory_record_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/memory_record_bean.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.common_func import convert_to_float +from msprof_analyze.compare_tools.compare_backend.utils.common_func import convert_to_float class MemoryRecordBean: diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/operator_memory_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/operator_memory_bean.py index 2d9d5c813f1e4495903719bcb491af6670d8e398..71e5548fef439f031cab3004128dd1d2a744855a 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/operator_memory_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/operator_memory_bean.py @@ -14,7 +14,7 @@ # limitations under the License. from decimal import Decimal -from compare_backend.utils.common_func import convert_to_float, convert_to_decimal +from msprof_analyze.compare_tools.compare_backend.utils.common_func import convert_to_float, convert_to_decimal class OperatorMemoryBean: diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py index 9afcab5ec03a9989b14b59d206b8ef8939839237..ab12d640a1aad9478ca067c56db3bcc10a156a0c 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py @@ -14,9 +14,8 @@ # limitations under the License. from decimal import Decimal -from compare_backend.utils.common_func import convert_to_float, convert_to_decimal -from compare_backend.compare_config.compare_config import CompareConfig - +from msprof_analyze.compare_tools.compare_backend.utils.common_func import convert_to_float, convert_to_decimal +from msprof_analyze.compare_tools.compare_backend.compare_config.compare_config import CompareConfig from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py index 0253fb61141ab3b269c5875c7e54049955ad8c3d..059416ec15e54b7732eb40cbf8b0e6ef1227bf90 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py @@ -14,10 +14,9 @@ # limitations under the License. from math import isclose -from compare_backend.compare_bean.profiling_info import ProfilingInfo -from compare_backend.utils.common_func import calculate_diff_ratio -from compare_backend.utils.excel_config import ExcelConfig, CellFormatType - +from msprof_analyze.compare_tools.compare_backend.compare_bean.profiling_info import ProfilingInfo +from msprof_analyze.compare_tools.compare_backend.utils.common_func import calculate_diff_ratio +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig, CellFormatType from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/compare_config/compare_config.py b/profiler/msprof_analyze/compare_tools/compare_backend/compare_config/compare_config.py index a44c074ea6b946e051a38535b0481564f07aef9b..4d044a1fa010878e69e13d3a1fcd79a6e12053dc 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/compare_config/compare_config.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/compare_config/compare_config.py @@ -14,8 +14,7 @@ # limitations under the License. import os -from compare_backend.utils.singleton import Singleton - +from msprof_analyze.compare_tools.compare_backend.utils.singleton import Singleton from msprof_analyze.prof_common.utils import SafeConfigReader diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/comparison_generator.py b/profiler/msprof_analyze/compare_tools/compare_backend/comparison_generator.py index 4a880822aed46631e16babaff15cfc68894cfc21..2c3c3d920ce3896336f396eb51d4df3920ded7d7 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/comparison_generator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/comparison_generator.py @@ -12,13 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.generator.detail_performance_generator import DetailPerformanceGenerator -from compare_backend.generator.overall_performance_generator import OverallPerformanceGenerator -from compare_backend.interface.overall_interface import OverallInterface -from compare_backend.interface.compare_interface import CompareInterface -from compare_backend.profiling_parser.gpu_profiling_parser import GPUProfilingParser -from compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingParser -from compare_backend.utils.args_manager import ArgsManager +from msprof_analyze.compare_tools.compare_backend.generator.detail_performance_generator \ + import DetailPerformanceGenerator +from msprof_analyze.compare_tools.compare_backend.generator.overall_performance_generator \ + import OverallPerformanceGenerator +from msprof_analyze.compare_tools.compare_backend.interface.overall_interface import OverallInterface +from msprof_analyze.compare_tools.compare_backend.interface.compare_interface import CompareInterface +from msprof_analyze.compare_tools.compare_backend.profiling_parser.gpu_profiling_parser import GPUProfilingParser +from msprof_analyze.compare_tools.compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingParser +from msprof_analyze.compare_tools.compare_backend.utils.args_manager import ArgsManager from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.additional_args_manager import AdditionalArgsManager from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/module_data_prepare.py b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/module_data_prepare.py index 52df18c78b82455c5c68b66e766ac7aff090c9d5..e8c56b26080c43780b81bffc2f0d61ccde5c3f64 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/module_data_prepare.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/module_data_prepare.py @@ -15,11 +15,10 @@ import copy from queue import Queue -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.profiling_parser.base_profiling_parser import ProfilingResult -from compare_backend.utils.module_node import ModuleNode -from compare_backend.utils.tree_builder import TreeBuilder - +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import ProfilingResult +from msprof_analyze.compare_tools.compare_backend.utils.module_node import ModuleNode +from msprof_analyze.compare_tools.compare_backend.utils.tree_builder import TreeBuilder from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/operator_data_prepare.py b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/operator_data_prepare.py index 003a023221257d406047dbb9797af403f21b8512..b5da970126ca2d652875974cca616beada3814fb 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/operator_data_prepare.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/operator_data_prepare.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.profiling_parser.base_profiling_parser import ProfilingResult -from compare_backend.utils.tree_builder import TreeBuilder - +from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import ProfilingResult +from msprof_analyze.compare_tools.compare_backend.utils.tree_builder import TreeBuilder from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py index b8e674fe987e63f29cd0459ccf3f2f762910da5f..5c2590c723e646660b456acbdf3f114fb2726190 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py @@ -14,11 +14,10 @@ # limitations under the License. from collections import deque -from compare_backend.utils.name_function import NameFunction -from compare_backend.utils.common_func import longest_common_subsequence_matching -from compare_backend.utils.torch_op_node import TorchOpNode -from compare_backend.utils.module_node import ModuleNode - +from msprof_analyze.compare_tools.compare_backend.utils.name_function import NameFunction +from msprof_analyze.compare_tools.compare_backend.utils.common_func import longest_common_subsequence_matching +from msprof_analyze.compare_tools.compare_backend.utils.torch_op_node import TorchOpNode +from msprof_analyze.compare_tools.compare_backend.utils.module_node import ModuleNode from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/disaggregate/overall_perf_interface.py b/profiler/msprof_analyze/compare_tools/compare_backend/disaggregate/overall_perf_interface.py index c59b35dd66143ef1ddff54e5176300a5592cf5d0..4dfbe559210e80c5072e1ed076253c483ef285e6 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/disaggregate/overall_perf_interface.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/disaggregate/overall_perf_interface.py @@ -12,11 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.profiling_parser.gpu_profiling_parser import GPUProfilingParser -from compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingParser -from compare_backend.utils.args_manager import ArgsManager -from compare_backend.utils.compare_args import Args - +from msprof_analyze.compare_tools.compare_backend.profiling_parser.gpu_profiling_parser import GPUProfilingParser +from msprof_analyze.compare_tools.compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingParser +from msprof_analyze.compare_tools.compare_backend.utils.args_manager import ArgsManager +from msprof_analyze.compare_tools.compare_backend.utils.compare_args import Args from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger from msprof_analyze.prof_common.path_manager import PathManager diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/generator/detail_performance_generator.py b/profiler/msprof_analyze/compare_tools/compare_backend/generator/detail_performance_generator.py index d48353be9c826330409790a49357fa3d58c7366c..6479d624fae6d3c7aa437d7f152dde7284b74f43 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/generator/detail_performance_generator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/generator/detail_performance_generator.py @@ -16,31 +16,32 @@ import os from collections import OrderedDict from datetime import datetime -from compare_backend.comparator.communication_comparator import CommunicationComparator -from compare_backend.comparator.module_comparetor import ModuleComparator -from compare_backend.comparator.module_statistic_comparator import ModuleStatisticComparator -from compare_backend.comparator.operator_comparator import OperatorComparator -from compare_backend.comparator.operator_statistic_comparator import OperatorStatisticComparator -from compare_backend.comparator.api_compare_comparator import ApiCompareComparator -from compare_backend.comparator.kernel_compare_comparator import KernelCompareComparator -from compare_backend.comparator.overall_metrics_comparator import OverallMetricsComparator -from compare_backend.compare_bean.communication_bean import CommunicationBean -from compare_backend.compare_bean.memory_compare_bean import MemoryCompareBean -from compare_backend.compare_bean.memory_statistic_bean import MemoryStatisticBean -from compare_backend.compare_bean.module_compare_bean import ModuleCompareBean -from compare_backend.compare_bean.module_statistic_bean import ModuleStatisticBean -from compare_backend.compare_bean.operator_compare_bean import OperatorCompareBean -from compare_backend.compare_bean.operator_statistic_bean import OperatorStatisticBean -from compare_backend.compare_bean.api_compare_bean import ApiCompareBean -from compare_backend.compare_bean.kernel_compare_bean import KernelCompareBean -from compare_backend.compare_bean.overall_metrics_bean import OverallMetricsBean -from compare_backend.data_prepare.module_data_prepare import ModuleDataPrepare -from compare_backend.data_prepare.operator_data_prepare import OperatorDataPrepare -from compare_backend.comparator.kernel_type_comparator import KernelTypeComparator -from compare_backend.compare_bean.kernel_type_compare_bean import KernelTypeCompareBean -from compare_backend.view.excel_view import ExcelView -from compare_backend.data_prepare.sequence_pre_matching import SequencePreMatching - +from msprof_analyze.compare_tools.compare_backend.comparator.communication_comparator import CommunicationComparator +from msprof_analyze.compare_tools.compare_backend.comparator.module_comparetor import ModuleComparator +from msprof_analyze.compare_tools.compare_backend.comparator.module_statistic_comparator \ + import ModuleStatisticComparator +from msprof_analyze.compare_tools.compare_backend.comparator.operator_comparator import OperatorComparator +from msprof_analyze.compare_tools.compare_backend.comparator.operator_statistic_comparator \ + import OperatorStatisticComparator +from msprof_analyze.compare_tools.compare_backend.comparator.api_compare_comparator import ApiCompareComparator +from msprof_analyze.compare_tools.compare_backend.comparator.kernel_compare_comparator import KernelCompareComparator +from msprof_analyze.compare_tools.compare_backend.comparator.overall_metrics_comparator import OverallMetricsComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.communication_bean import CommunicationBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.memory_compare_bean import MemoryCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.memory_statistic_bean import MemoryStatisticBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.module_compare_bean import ModuleCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.module_statistic_bean import ModuleStatisticBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.operator_compare_bean import OperatorCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.operator_statistic_bean import OperatorStatisticBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.api_compare_bean import ApiCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.kernel_compare_bean import KernelCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.overall_metrics_bean import OverallMetricsBean +from msprof_analyze.compare_tools.compare_backend.data_prepare.module_data_prepare import ModuleDataPrepare +from msprof_analyze.compare_tools.compare_backend.data_prepare.operator_data_prepare import OperatorDataPrepare +from msprof_analyze.compare_tools.compare_backend.comparator.kernel_type_comparator import KernelTypeComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.kernel_type_compare_bean import KernelTypeCompareBean +from msprof_analyze.compare_tools.compare_backend.view.excel_view import ExcelView +from msprof_analyze.compare_tools.compare_backend.data_prepare.sequence_pre_matching import SequencePreMatching from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/generator/overall_performance_generator.py b/profiler/msprof_analyze/compare_tools/compare_backend/generator/overall_performance_generator.py index a162353e89a4fe1eec120ae9d9f32520df4162fc..e144ef185c433543f1b05afd64a8125d0d1f26c4 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/generator/overall_performance_generator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/generator/overall_performance_generator.py @@ -12,11 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.overall_performance_comparator import OverallPerformanceComparator -from compare_backend.compare_bean.profiling_info import ProfilingInfo -from compare_backend.generator.base_generator import BaseGenerator -from compare_backend.view.screen_view import ScreenView - +from msprof_analyze.compare_tools.compare_backend.comparator.overall_performance_comparator \ + import OverallPerformanceComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.profiling_info import ProfilingInfo +from msprof_analyze.compare_tools.compare_backend.generator.base_generator import BaseGenerator +from msprof_analyze.compare_tools.compare_backend.view.screen_view import ScreenView from msprof_analyze.prof_common.logger import get_logger logger = get_logger() diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/interface/compare_interface.py b/profiler/msprof_analyze/compare_tools/compare_backend/interface/compare_interface.py index 58e4a0dbebbaf74a606ad5b605f67ff56c350060..2a459c7621a22c94411a3b93a46ff8c877f6bdea 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/interface/compare_interface.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/interface/compare_interface.py @@ -12,17 +12,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.operator_comparator import OperatorComparator -from compare_backend.comparator.api_compare_comparator import ApiCompareComparator -from compare_backend.comparator.kernel_compare_comparator import KernelCompareComparator -from compare_backend.compare_bean.operator_compare_bean import OperatorCompareBean -from compare_backend.compare_bean.api_compare_bean import ApiCompareBean -from compare_backend.compare_bean.kernel_compare_bean import KernelCompareBean -from compare_backend.data_prepare.operator_data_prepare import OperatorDataPrepare -from compare_backend.data_prepare.sequence_pre_matching import SequencePreMatching -from compare_backend.comparator.kernel_type_comparator import KernelTypeComparator -from compare_backend.compare_bean.kernel_type_compare_bean import KernelTypeCompareBean - +from msprof_analyze.compare_tools.compare_backend.comparator.operator_comparator import OperatorComparator +from msprof_analyze.compare_tools.compare_backend.comparator.api_compare_comparator import ApiCompareComparator +from msprof_analyze.compare_tools.compare_backend.comparator.kernel_compare_comparator import KernelCompareComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.operator_compare_bean import OperatorCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.api_compare_bean import ApiCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.kernel_compare_bean import KernelCompareBean +from msprof_analyze.compare_tools.compare_backend.data_prepare.operator_data_prepare import OperatorDataPrepare +from msprof_analyze.compare_tools.compare_backend.data_prepare.sequence_pre_matching import SequencePreMatching +from msprof_analyze.compare_tools.compare_backend.comparator.kernel_type_comparator import KernelTypeComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.kernel_type_compare_bean import KernelTypeCompareBean from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/interface/overall_interface.py b/profiler/msprof_analyze/compare_tools/compare_backend/interface/overall_interface.py index ab1f2b23d0b61f3e35a120f7c794cf2aaa720c56..dcd7c2b49e8b279cdef1d822dfc43e98366e3f3a 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/interface/overall_interface.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/interface/overall_interface.py @@ -12,8 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.comparator.overall_performance_comparator import OverallPerformanceComparator -from compare_backend.compare_bean.profiling_info import ProfilingInfo +from msprof_analyze.compare_tools.compare_backend.comparator.overall_performance_comparator \ + import OverallPerformanceComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.profiling_info import ProfilingInfo from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py index c9a31a926931b5e06345d9f15bbc739e42db38e2..b3d9a29944fe7f722b2fc54ac0381f8c23c1af14 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py @@ -17,10 +17,14 @@ from decimal import Decimal import ijson -from compare_backend.compare_bean.origin_data_bean.compare_event import KernelEvent, MemoryEvent -from compare_backend.compare_bean.origin_data_bean.kernel_details_bean import KernelDetailsBean -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.compare_bean.profiling_info import ProfilingInfo +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.compare_event import ( + KernelEvent, + MemoryEvent +) +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.kernel_details_bean \ + import KernelDetailsBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.profiling_info import ProfilingInfo from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import FileManager from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index b69950deede1235e4c3edcd283292b2e33e13d83..f5a6e564c575941cdb2025e39d73eaea3fd8d5b3 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -15,8 +15,8 @@ import sys from collections import defaultdict, Counter -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.profiling_parser.base_profiling_parser import BaseProfilingParser +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import BaseProfilingParser from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 941acedf8091559a4700ac9cc2903869f69508f3..6e7e9e1c1e5963303ceb2cd73abda2089d7ae069 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -16,12 +16,14 @@ import os import sys from math import ceil -from compare_backend.compare_bean.origin_data_bean.kernel_details_bean import KernelDetailsBean -from compare_backend.compare_bean.origin_data_bean.memory_record_bean import MemoryRecordBean -from compare_backend.compare_bean.origin_data_bean.operator_memory_bean import OperatorMemoryBean -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.profiling_parser.base_profiling_parser import BaseProfilingParser - +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.kernel_details_bean \ + import KernelDetailsBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.memory_record_bean \ + import MemoryRecordBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.operator_memory_bean \ + import OperatorMemoryBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import BaseProfilingParser from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.op_stastic_bean import OpStatisticBean from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import FileManager diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/utils/args_manager.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/args_manager.py index 320606c5bf50accd41ae47c6fb3bd27e0a2177ed..6ac463982b434901390b4440ef0bcde956d62362 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/utils/args_manager.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/utils/args_manager.py @@ -15,7 +15,7 @@ import os.path import re -from compare_backend.utils.singleton import Singleton +from msprof_analyze.compare_tools.compare_backend.utils.singleton import Singleton from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.file_manager import FileManager from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/utils/module_node.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/module_node.py index 12e2f7a30d001d3dec7115c85a680e8dc4e074cb..98fa2ca1292874f5a4b3dd6903ae48552e52e1b6 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/utils/module_node.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/utils/module_node.py @@ -15,8 +15,8 @@ import re from math import ceil -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.utils.torch_op_node import TorchOpNode +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.utils.torch_op_node import TorchOpNode class ModuleNode: diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/utils/name_function.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/name_function.py index 8068f52f4212c13300fd2ff6f63cc5d09fe700c2..f1e2e90d22bb6d86b73ee5c43315a7677741d1a8 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/utils/name_function.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/utils/name_function.py @@ -12,8 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.utils.module_node import ModuleNode -from compare_backend.utils.torch_op_node import TorchOpNode +from msprof_analyze.compare_tools.compare_backend.utils.module_node import ModuleNode +from msprof_analyze.compare_tools.compare_backend.utils.torch_op_node import TorchOpNode class NameFunction: diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/utils/torch_op_node.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/torch_op_node.py index 9692241a961d763f9ae117d0523d337bf9f96fec..2b72ad3d990f1bb1a73aa071d359b32d269934b1 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/utils/torch_op_node.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/utils/torch_op_node.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from compare_backend.compare_bean.origin_data_bean.compare_event import MemoryEvent -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean - +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.compare_event import MemoryEvent +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/utils/tree_builder.py b/profiler/msprof_analyze/compare_tools/compare_backend/utils/tree_builder.py index 5a3e10aa42c76f4f14f69f7f4bd32acff938495b..6872bed55f67df6e5736be9cdd0ade0d97f4f598 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/utils/tree_builder.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/utils/tree_builder.py @@ -15,9 +15,9 @@ from queue import Queue -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.utils.module_node import ModuleNode -from compare_backend.utils.torch_op_node import TorchOpNode +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.utils.module_node import ModuleNode +from msprof_analyze.compare_tools.compare_backend.utils.torch_op_node import TorchOpNode class TreeBuilder: diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/view/excel_view.py b/profiler/msprof_analyze/compare_tools/compare_backend/view/excel_view.py index 7cf3a8554c0e5bd65a8f3f0daccefdaf3cfff65c..6a094fdf3df8828f0a89e3333517459257440d19 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/view/excel_view.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/view/excel_view.py @@ -16,8 +16,8 @@ import os from xlsxwriter import Workbook -from compare_backend.view.base_view import BaseView -from compare_backend.view.work_sheet_creator import WorkSheetCreator +from msprof_analyze.compare_tools.compare_backend.view.base_view import BaseView +from msprof_analyze.compare_tools.compare_backend.view.work_sheet_creator import WorkSheetCreator from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/view/screen_view.py b/profiler/msprof_analyze/compare_tools/compare_backend/view/screen_view.py index fb615979d27d19caa3a8838f211f2cc7162fb94c..5797d0bf8c4a64336d025043e41f5b7cec1e0054 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/view/screen_view.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/view/screen_view.py @@ -14,7 +14,7 @@ # limitations under the License. from prettytable import PrettyTable -from compare_backend.view.base_view import BaseView +from msprof_analyze.compare_tools.compare_backend.view.base_view import BaseView class ScreenView(BaseView): diff --git a/profiler/msprof_analyze/compare_tools/compare_backend/view/work_sheet_creator.py b/profiler/msprof_analyze/compare_tools/compare_backend/view/work_sheet_creator.py index 250a41cff3ec51b33aa994699989aa3e891f4b89..b73f6df97e81886e6034eae0dcdbe4c180f7995c 100644 --- a/profiler/msprof_analyze/compare_tools/compare_backend/view/work_sheet_creator.py +++ b/profiler/msprof_analyze/compare_tools/compare_backend/view/work_sheet_creator.py @@ -14,7 +14,7 @@ # limitations under the License. from xlsxwriter import Workbook -from compare_backend.utils.excel_config import ExcelConfig, CellFormatType +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig, CellFormatType class WorkSheetCreator: diff --git a/profiler/msprof_analyze/compare_tools/compare_interface/comparison_interface.py b/profiler/msprof_analyze/compare_tools/compare_interface/comparison_interface.py index bb3a18eb4b2a4b258abd0cf1a8a0f5afb169494f..70da42d20e250834477d811a94aa749fae380b33 100644 --- a/profiler/msprof_analyze/compare_tools/compare_interface/comparison_interface.py +++ b/profiler/msprof_analyze/compare_tools/compare_interface/comparison_interface.py @@ -12,16 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import sys -import os - -sys.path.append( - os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "cluster_analyse")) -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from compare_backend.comparison_generator import ComparisonGenerator -from compare_backend.disaggregate.overall_perf_interface import OverallPerfInterface -from compare_backend.utils.compare_args import Args +from msprof_analyze.compare_tools.compare_backend.comparison_generator import ComparisonGenerator +from msprof_analyze.compare_tools.compare_backend.disaggregate.overall_perf_interface import OverallPerfInterface +from msprof_analyze.compare_tools.compare_backend.utils.compare_args import Args from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.analyze_dict import AnalyzeDict from msprof_analyze.prof_common.logger import get_logger diff --git a/profiler/msprof_analyze/compare_tools/performance_compare.py b/profiler/msprof_analyze/compare_tools/performance_compare.py index a8fc94a8ee509e6bc8fc3d5572d3875062b0b0f0..478613e33c2953cc5978b1a5eef35b58cfde7945 100644 --- a/profiler/msprof_analyze/compare_tools/performance_compare.py +++ b/profiler/msprof_analyze/compare_tools/performance_compare.py @@ -15,14 +15,8 @@ import argparse import ast import datetime -import os.path -import sys -sys.path.append( - os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "cluster_analyse")) -sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) - -from compare_backend.comparison_generator import ComparisonGenerator +from msprof_analyze.compare_tools.compare_backend.comparison_generator import ComparisonGenerator from msprof_analyze.prof_common.analyze_dict import AnalyzeDict from msprof_analyze.prof_common.logger import get_logger from msprof_analyze.prof_common.path_manager import PathManager diff --git a/profiler/msprof_analyze/prof_common/utils.py b/profiler/msprof_analyze/prof_common/utils.py index f6a9f8618c92473a362070ba3a6b113ecc4e5628..7794d6f9858e4b96fc37f4768983a400113a0b11 100644 --- a/profiler/msprof_analyze/prof_common/utils.py +++ b/profiler/msprof_analyze/prof_common/utils.py @@ -12,14 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import sys +import os + +sys.path.append( + os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "cluster_analyse")) +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + import configparser import os from email.utils import parseaddr from typing import Dict, List from urllib.parse import urlparse -from .logger import get_logger -from .path_manager import PathManager +from msprof_analyze.prof_common.logger import get_logger +from msprof_analyze.prof_common.path_manager import PathManager logger = get_logger() diff --git a/profiler/msprof_analyze/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py b/profiler/msprof_analyze/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py index c0e6de210826ba321b019b075c0db19478dd86ba..a485d62188deaa661499b79b94c94ce990398fcd 100644 --- a/profiler/msprof_analyze/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py +++ b/profiler/msprof_analyze/test/st/advisor/test_advisor_cmd_single_ascend_pt_compare.py @@ -22,7 +22,7 @@ import pandas as pd from bs4 import BeautifulSoup from msprof_analyze.prof_common.path_manager import PathManager -from .utils import get_files, execute_cmd +from msprof_analyze.test.st.advisor.utils import get_files, execute_cmd class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): diff --git a/profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py b/profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py index ba9cd68062aa5d5dc5cc52c9706bd279d8827c35..bbc07adebfb5fd96da6f61cdd9a24e107b3653c8 100644 --- a/profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py +++ b/profiler/msprof_analyze/test/st/cluster_analyse/test_cluster_analyse_pytorch_db.py @@ -22,10 +22,13 @@ import pandas as pd from msprof_analyze.test.st.utils import execute_cmd, select_count, select_by_query from msprof_analyze.prof_common.file_manager import FileManager from msprof_analyze.prof_common.path_manager import PathManager -from .cluster_communication_analyzer_bandwidth_db import ClusterCommunicationAnalyzerBandwidthDb -from .cluster_communication_analyzer_matrix_db import ClusterCommunicationAnalyzerMatrixDb -from .cluster_communication_analyzer_time_db import ClusterCommunicationAnalyzerTime -from .cluster_step_trace_time_db import ClusterStepTraceTimeDb +from msprof_analyze.test.st.cluster_analyse.cluster_communication_analyzer_bandwidth_db \ + import ClusterCommunicationAnalyzerBandwidthDb +from msprof_analyze.test.st.cluster_analyse.cluster_communication_analyzer_matrix_db \ + import ClusterCommunicationAnalyzerMatrixDb +from msprof_analyze.test.st.cluster_analyse.cluster_communication_analyzer_time_db \ + import ClusterCommunicationAnalyzerTime +from msprof_analyze.test.st.cluster_analyse.cluster_step_trace_time_db import ClusterStepTraceTimeDb class TestClusterAnalysePytorchDb(TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_advice_base.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_advice_base.py index d22c33ea05ef9955c86cea5751795501accc8c02..a86cd57af1d1992675308295ba2eaf5f60303b31 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_advice_base.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_advice_base.py @@ -18,7 +18,7 @@ import unittest from unittest import mock from unittest.mock import MagicMock -from advisor_backend.cluster_advice.cluster_advice_base import ClusterAdviceBase +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_advice_base import ClusterAdviceBase class MockChildClusterAdvice(ClusterAdviceBase): @@ -70,7 +70,8 @@ class TestClusterAdviceBase(unittest.TestCase): def test_cluster_analyze_normal(self): mock_inst = MockChildClusterAdvice(self.tmp_dir) - with mock.patch("advisor_backend.cluster_advice.cluster_advice_base.Interface") as mock_if: + with mock.patch("msprof_analyze.advisor.advisor_backend.cluster_advice." + "cluster_advice_base.Interface") as mock_if: mock_if_inst = mock_if.return_value mock_if_inst.run = MagicMock(name="run") mock_inst.cluster_analyze() @@ -79,7 +80,8 @@ class TestClusterAdviceBase(unittest.TestCase): def test_cluster_analyze_abnormal(self): mock_inst = MockChildClusterAdvice(self.tmp_dir) with self.assertRaises(ValueError): - with mock.patch("advisor_backend.cluster_advice.cluster_advice_base.Interface") as mock_if: + with mock.patch("msprof_analyze.advisor.advisor_backend.cluster_advice." + "cluster_advice_base.Interface") as mock_if: mock_if_inst = mock_if.return_value mock_if_inst.run = mock.Mock(name="run", side_effect=Exception('Error!')) mock_inst.cluster_analyze() diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py index 5d6b51df2daf2f2a2ca99d37ee8277de1f285314..8f6d2a6a50329bfabe1fd0d92d50f20571379fb2 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_cluster_pipeline_advice.py @@ -17,10 +17,10 @@ from unittest import mock from collections import deque from collections import defaultdict -from advisor_backend.cluster_advice.cluster_pipeline_advice import ClusterPipelineAdvice -from advisor_backend.cluster_advice.cluster_pipeline_advice import FineTraceViewData -from advisor_backend.cluster_advice.cluster_pipeline_advice import PipelineTimeSlice -from advisor_backend.cluster_advice.cluster_pipeline_advice import PipelineTraceViewer +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_pipeline_advice import ClusterPipelineAdvice +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_pipeline_advice import FineTraceViewData +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_pipeline_advice import PipelineTimeSlice +from msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_pipeline_advice import PipelineTraceViewer class TestClusterPipelineAdvice(unittest.TestCase): @@ -119,7 +119,8 @@ class TestClusterPipelineAdvice(unittest.TestCase): bp_op2 = {"ph": "X", "name": "autogard::add", "ts": str(2000000000 - 100), "dur": 2000, "tid": 3, "pid": 1, "args": {}} res_bp_ops = [(bp_op1, bp_op2)] - with mock.patch('advisor_backend.cluster_advice.cluster_pipeline_advice.ClusterPipelineAdvice.double_queue_pop', + with mock.patch('msprof_analyze.advisor.advisor_backend.cluster_advice.cluster_pipeline_advice.' + 'ClusterPipelineAdvice.double_queue_pop', return_value=(None, res_bp_ops)): advice = ClusterPipelineAdvice('./tmp_dir', {}) res_check = advice.get_fp_bp_bound_ops(fine_data) diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py index 724c51fb3e8dfd1933d547ce7fc24b3f147a9a9e..f114b2eb7387fe179eebf86cd0526fe6721e7198 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_kernel_cluster_advice.py @@ -20,7 +20,7 @@ from unittest import mock from unittest.mock import MagicMock from msprof_analyze.prof_common.constant import Constant -from advisor_backend.cluster_advice.kernel_cluster_advice import KernelClusterAdvice +from msprof_analyze.advisor.advisor_backend.cluster_advice.kernel_cluster_advice import KernelClusterAdvice class TestClusterAdviceBase(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_link_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_link_advice.py index e711ca01837dffcd3fcc25dde059129e6c79a3a4..3a0b3de50fb1a9896d88a22915991274b6466992 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_link_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_link_advice.py @@ -14,7 +14,7 @@ # limitations under the License. import unittest -from advisor_backend.cluster_advice.slow_link_advice import SlowLinkAdvice +from msprof_analyze.advisor.advisor_backend.cluster_advice.slow_link_advice import SlowLinkAdvice class TestSlowLinkAdvice(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_rank_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_rank_advice.py index b5900f7c91ddb2e5205ec670bad6a630ed463701..8c196faba0708ad8be2f23445c645d268b16642e 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_rank_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/cluster_advice/test_slow_rank_advice.py @@ -14,7 +14,7 @@ # limitations under the License. import unittest -from advisor_backend.cluster_advice.slow_rank_advice import SlowRankAdvice +from msprof_analyze.advisor.advisor_backend.cluster_advice.slow_rank_advice import SlowRankAdvice class TestSlowRankAdvice(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npu_slow_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npu_slow_advice.py index 799a673d26ea90fa90cbf7558532c7d6f76c404e..bfbb17df5bc5c8b6626764a1f963be883560a111 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npu_slow_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npu_slow_advice.py @@ -19,8 +19,8 @@ import stat import csv import unittest -from advisor_backend.interface import Interface -from advisor_backend.compute_advice.npu_slow_advice import NpuSlowAdvice +from msprof_analyze.advisor.advisor_backend.interface import Interface +from msprof_analyze.advisor.advisor_backend.compute_advice.npu_slow_advice import NpuSlowAdvice class TestNpuSlowAdvice(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npufused_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npufused_advice.py index f24066e5c9f56f19112ab6968f169925710cb74e..4670882c05faed698bdd35842c385a0c4f02e2d0 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npufused_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/compute_advice/test_npufused_advice.py @@ -18,9 +18,8 @@ import shutil import stat import csv import unittest -import pytest -from advisor_backend.interface import Interface +from msprof_analyze.advisor.advisor_backend.interface import Interface class TestComputeAdvice(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py index 93781aac7dbb1e98e9d2c020b14b446fd5b9601a..d63176dc1d399c5f94703c3aa8a2a522143bcfc7 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/prof_bean_advisor/test_cluster_step_trace_time_bean.py @@ -16,10 +16,9 @@ import os import stat import shutil import unittest -from unittest import mock -from unittest.mock import MagicMock -from advisor_backend.prof_bean_advisor.cluster_step_trace_time_bean import ClusterStepTraceTimeBean +from msprof_analyze.advisor.advisor_backend.prof_bean_advisor.cluster_step_trace_time_bean \ + import ClusterStepTraceTimeBean class TestClusterStepTraceTimeBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_opsche_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_opsche_advice.py index eb2cabcb2822eb5ab55a369e1c8e518e421ad53a..5854ffa15ef29f5ce528a9e7d12d2429e6f30dd4 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_opsche_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_opsche_advice.py @@ -13,13 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -import shutil -import stat -import json import unittest -import pytest -from advisor_backend.interface import Interface +from msprof_analyze.advisor.advisor_backend.interface import Interface class TestOpScheAdvice(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_optimizer_advice.py b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_optimizer_advice.py index 8b4725af4375800856086943e03075ce6b9b8659..3ed142f7f7a966c6ca0aa7f797bcb3282c214fb8 100644 --- a/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_optimizer_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/advisor_backend/timeline_advice/test_optimizer_advice.py @@ -17,9 +17,8 @@ import shutil import stat import json import unittest -import pytest -from advisor_backend.interface import Interface +from msprof_analyze.advisor.advisor_backend.interface import Interface class TestOptimizerAdvice(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py index 115f283788da33806cc4e9eba1e49a5f323384ec..dbe153027baca7e5c8e8b8424fd04181ee60b3f4 100644 --- a/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py @@ -18,6 +18,7 @@ import stat import json import csv import unittest + from msprof_analyze.advisor.interface.interface import Interface from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes diff --git a/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_byte_alignment_analyzer.py b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_byte_alignment_analyzer.py index 3fd018a9768d9050fcef52a309f7fc4835e933b9..9b4f02b41701d69a6cb539427df79f66bf24d143 100644 --- a/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_byte_alignment_analyzer.py +++ b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_byte_alignment_analyzer.py @@ -17,6 +17,7 @@ import shutil import stat import json import unittest + from msprof_analyze.advisor.interface.interface import Interface from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes diff --git a/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_packet_advice.py b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_packet_advice.py index 39905ac40c8aa49135c0291078d27d962a88d700..34c3e2acb6405f47779936f07a0758986b32790a 100644 --- a/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_packet_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_packet_advice.py @@ -16,8 +16,8 @@ import os import shutil import stat import json - import unittest + from msprof_analyze.advisor.interface.interface import Interface from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes diff --git a/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py index 6c858a4f087c5869760c52e7cc955e162b62884c..aec8bd6b032eb9b139626ecaf5129e35ed6128c8 100644 --- a/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py @@ -16,8 +16,8 @@ import os import shutil import stat import json - import unittest + from msprof_analyze.advisor.interface.interface import Interface from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes diff --git a/profiler/msprof_analyze/test/ut/advisor/compute_advice/test_frequency_advice.py b/profiler/msprof_analyze/test/ut/advisor/compute_advice/test_frequency_advice.py index 0f32fdf97e886e28f35b8b5a5f16898ceec538f7..f316032e4ae70a96fbec4cadf105c4436771d09f 100644 --- a/profiler/msprof_analyze/test/ut/advisor/compute_advice/test_frequency_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/compute_advice/test_frequency_advice.py @@ -16,8 +16,8 @@ import os import shutil import stat import json - import unittest + from msprof_analyze.advisor.interface.interface import Interface from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes from msprof_analyze.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset diff --git a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_conjectured_gc_advice.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_conjectured_gc_advice.py index f99aa946b8ac03ad49414578cc55df8d338c7932..ad5b76c84758f9e3fe5b37e76a5b785f1e21a0d7 100644 --- a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_conjectured_gc_advice.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_conjectured_gc_advice.py @@ -18,6 +18,7 @@ import stat import json import multiprocessing import unittest + from msprof_analyze.advisor.interface.interface import Interface from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes diff --git a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_dataloader_checker.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_dataloader_checker.py index 4c160d8a1b3fe6dc1b8e70d5e332fe0aef918b21..5ad2b5b26423939e5f9a7c9c853a6571ff6e8b88 100644 --- a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_dataloader_checker.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_dataloader_checker.py @@ -14,7 +14,6 @@ # limitations under the License. import unittest import os -import sys import yaml from msprof_analyze.advisor.analyzer.dataloader.dataloader_checker import DataloaderChecker diff --git a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_gc_checker.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_gc_checker.py index d67496a1948f4cade8ba242a362aa4ec26612237..b570813b1e28fd9a00b0d6fafb64c3cdc2f2722e 100644 --- a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_gc_checker.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_gc_checker.py @@ -18,6 +18,7 @@ import stat import json import multiprocessing import unittest + from msprof_analyze.advisor.interface.interface import Interface from msprof_analyze.advisor.common.analyzer_scopes import SupportedScopes diff --git a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_memory_op_checker.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_memory_op_checker.py index 573f06257ed0145efe133409f69dc0a16920f083..e267bbf9f6c1895c735f3fd90a12843913ec94f7 100644 --- a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_memory_op_checker.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_memory_op_checker.py @@ -14,7 +14,6 @@ # limitations under the License. import unittest import os -import sys import yaml from msprof_analyze.advisor.analyzer.memory.memory_checker import MemoryOpsChecker diff --git a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_syncbn_checker.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_syncbn_checker.py index 9ce89c41e52078ef0a77f5c81d190a34cc5ff470..0859bc1866727fe5c832686f64e4ffe5efe05512 100644 --- a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_syncbn_checker.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_syncbn_checker.py @@ -14,7 +14,6 @@ # limitations under the License. import unittest import os -import sys import yaml from msprof_analyze.advisor.analyzer.schedule.syncbn.syncbn_checker import SyncBNChecker diff --git a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_synchronize_stream.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_synchronize_stream.py index cc110e4d153a423c6d0ba5bee54545f7ab435d54..0a379b8b8d8646fb1ddc12f83b0645978dfe7945 100644 --- a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_synchronize_stream.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_synchronize_stream.py @@ -14,7 +14,6 @@ # limitations under the License. import unittest import os -import sys import yaml from msprof_analyze.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_checker import SynchronizeStreamChecker diff --git a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_timeline_op_collector.py b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_timeline_op_collector.py index e2c9121624b36ed056cadbff4e694a8163947dad..edef567259f8778896e6f3d3291fb4649664aecc 100644 --- a/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_timeline_op_collector.py +++ b/profiler/msprof_analyze/test/ut/advisor/timeline_advice/test_timeline_op_collector.py @@ -13,9 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import unittest -import os -import sys -import yaml from msprof_analyze.advisor.dataset.timeline_op_collector.timeline_op_collector import ( OpCompileCollector, diff --git a/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_pytorch_data_preprocessor.py b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_pytorch_data_preprocessor.py index 8292ee13890644eadd62ed38d9c4df5991c57af3..ee159cca792cb53d9504accef54be6ed93f9a980 100644 --- a/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_pytorch_data_preprocessor.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_pytorch_data_preprocessor.py @@ -19,7 +19,7 @@ import shutil import unittest from unittest import mock -from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor +from msprof_analyze.cluster_analyse.cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor class TestPytorchDataPreprocessor(unittest.TestCase): @@ -59,7 +59,8 @@ class TestPytorchDataPreprocessor(unittest.TestCase): def test_get_data_map_given_cluster_files_expect_rank_12(self): check = PytorchDataPreprocessor(self.dirs) - with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", return_value={}): + with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", + return_value={}): ret = check.get_data_map() self.assertIn(1, ret.keys()) self.assertIn(2, ret.keys()) diff --git a/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_step_trace_time_analysis.py b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_step_trace_time_analysis.py index 018a4f4b0acaa5479269234a8484f546e2f60fd8..bd8e2da21f7421e3686293367b02428197386f6e 100644 --- a/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_step_trace_time_analysis.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_data_preprocess/test_step_trace_time_analysis.py @@ -13,13 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. - -import os import unittest -from unittest import mock -from analysis.step_trace_time_analysis import StepTraceTimeAnalysis -from prof_bean.step_trace_time_bean import StepTraceTimeBean +from msprof_analyze.cluster_analyse.analysis.step_trace_time_analysis import StepTraceTimeAnalysis +from msprof_analyze.cluster_analyse.prof_bean.step_trace_time_bean import StepTraceTimeBean from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_utils/test_parallel_strategy_calculator.py b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_utils/test_parallel_strategy_calculator.py index 629ecca1628cd493ce0a6cf18e174e692921a01c..7707fd761aba717df508a407af254292dda5969f 100644 --- a/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_utils/test_parallel_strategy_calculator.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/cluster_utils/test_parallel_strategy_calculator.py @@ -14,7 +14,7 @@ # limitations under the License. import unittest -from cluster_utils.parallel_strategy_calculator import ParallelStrategyCalculator +from msprof_analyze.cluster_analyse.cluster_utils.parallel_strategy_calculator import ParallelStrategyCalculator class TestParallelStrategyCalculator(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/cluster_analyse/common_func/test_file_manager.py b/profiler/msprof_analyze/test/ut/cluster_analyse/common_func/test_file_manager.py index 9828638d384ddd8fcd54c9bc3821e7ec16d33e15..c919e71ceea14867e1e8675f63f1408bd34497a9 100644 --- a/profiler/msprof_analyze/test/ut/cluster_analyse/common_func/test_file_manager.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/common_func/test_file_manager.py @@ -20,7 +20,7 @@ import unittest import pytest from msprof_analyze.prof_common.file_manager import FileManager -from prof_bean.step_trace_time_bean import StepTraceTimeBean +from msprof_analyze.cluster_analyse.prof_bean.step_trace_time_bean import StepTraceTimeBean class TestFileManager(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py b/profiler/msprof_analyze/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py index eea0d37f076f471922c7ffd63b21e5b12715d70b..517327b81117f100a1bf0f71edbe9ebd45ef605e 100644 --- a/profiler/msprof_analyze/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/communication_group/test_communication_group_generator.py @@ -13,12 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. - -import os import unittest from unittest import mock -from communication_group.communication_group_generator import CommunicationGroupGenerator +from msprof_analyze.cluster_analyse.communication_group.communication_group_generator import CommunicationGroupGenerator from msprof_analyze.prof_common.constant import Constant @@ -34,7 +32,8 @@ class TestCommunicationGroupGenerator(unittest.TestCase): check.collective_group_dict = { 'group1': {0} } - with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", return_value=True): + with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", + return_value=True): check.generate_p2p_communication_group() ret = {0} self.assertEqual(ret, set(check.communication_group[Constant.P2P][0])) @@ -45,7 +44,8 @@ class TestCommunicationGroupGenerator(unittest.TestCase): 'group1': {1, 2, 3, 4}, 'group2': {5, 6, 7, 8}, } - with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", return_value=True): + with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", + return_value=True): check.generate_p2p_communication_group() ret_a = {1, 2, 3, 4} ret_b = {5, 6, 7, 8} @@ -72,7 +72,8 @@ class TestCommunicationGroupGenerator(unittest.TestCase): 'group15': {15, 13}, 'group16': {15, 14} } - with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", return_value=True): + with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", + return_value=True): check.generate_p2p_communication_group() ret_a = {0, 1, 2, 3} ret_b = {4, 5, 6, 7} @@ -103,7 +104,8 @@ class TestCommunicationGroupGenerator(unittest.TestCase): 'group15': {12, 13, 14, 15}, 'group16': {12, 13, 14, 15} } - with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", return_value=True): + with mock.patch("msprof_analyze.prof_common.file_manager.FileManager.read_json_file", + return_value=True): check.generate_p2p_communication_group() ret_a = {0, 1, 2, 3, 4, 5, 6, 7} ret_b = {8, 9, 10, 11, 12, 13, 14, 15} diff --git a/profiler/msprof_analyze/test/ut/cluster_analyse/prof_bean/test_step_trace_time_bean.py b/profiler/msprof_analyze/test/ut/cluster_analyse/prof_bean/test_step_trace_time_bean.py index df38eefc9a01bbfbc63f99d64ed772c6fe581ec0..f7d81543b277db0f371e8b6c63d78697ccc4a091 100644 --- a/profiler/msprof_analyze/test/ut/cluster_analyse/prof_bean/test_step_trace_time_bean.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/prof_bean/test_step_trace_time_bean.py @@ -14,7 +14,7 @@ # limitations under the License. import unittest -from prof_bean.step_trace_time_bean import StepTraceTimeBean +from msprof_analyze.cluster_analyse.prof_bean.step_trace_time_bean import StepTraceTimeBean class TestStepTraceTimeBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/cluster_analyse/recipes/test_compute_op_sum.py b/profiler/msprof_analyze/test/ut/cluster_analyse/recipes/test_compute_op_sum.py index 2fffa30444ded2cb3b61cd102e573c5960f52785..314a213bce44d83dbedb1cfed221722467d5411e 100644 --- a/profiler/msprof_analyze/test/ut/cluster_analyse/recipes/test_compute_op_sum.py +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/recipes/test_compute_op_sum.py @@ -15,7 +15,8 @@ import unittest import pandas as pd -from recipes.compute_op_sum.compute_op_sum import ComputeOpSum + +from msprof_analyze.cluster_analyse.recipes.compute_op_sum.compute_op_sum import ComputeOpSum from msprof_analyze.prof_common.constant import Constant diff --git a/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_communication_comparator.py b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_communication_comparator.py index 3cd1884c226c802f91e8af88bf46759dbb6d5be3..f44b7b3ddf01998217949887505f71375c933e42 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_communication_comparator.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_communication_comparator.py @@ -1,7 +1,7 @@ import unittest -from compare_backend.comparator.communication_comparator import CommunicationComparator -from compare_backend.compare_bean.communication_bean import CommunicationBean +from msprof_analyze.compare_tools.compare_backend.comparator.communication_comparator import CommunicationComparator +from msprof_analyze.compare_tools.compare_backend.compare_bean.communication_bean import CommunicationBean class TestCommunicationComparator(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_comparator.py b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_comparator.py index cb51b5756c4a72f85b0a67d3da3e0864d54312e0..cff56a146c8ac2e62ff2059371251b0cdaa14004 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_comparator.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_comparator.py @@ -1,6 +1,6 @@ import unittest -from compare_backend.comparator.operator_comparator import OperatorComparator +from msprof_analyze.compare_tools.compare_backend.comparator.operator_comparator import OperatorComparator class MockBean: diff --git a/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py index 133fa197f14a5bd0a4ca9d7280b096962190a066..a123a98bffa44804154e6bc1caaa5fa07f38093a 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/comparator/test_operator_statistic_comparator.py @@ -1,7 +1,8 @@ import unittest from unittest.mock import patch -from compare_backend.comparator.operator_statistic_comparator import OperatorStatisticComparator +from msprof_analyze.compare_tools.compare_backend.comparator.operator_statistic_comparator \ + import OperatorStatisticComparator class MockBean: @@ -24,7 +25,8 @@ class TestOperatorStatisticComparator(unittest.TestCase): base_dict = {"add": [1], "matmul": [1]} comparison_dict = {"add": [1], "reduce": [1]} with patch( - "compare_backend.comparator.operator_statistic_comparator.OperatorStatisticComparator._group_by_op_name", + "msprof_analyze.compare_tools.compare_backend.comparator.operator_statistic_comparator." + "OperatorStatisticComparator._group_by_op_name", return_value=(base_dict, comparison_dict)): comparator = OperatorStatisticComparator({1: 2}, MockBean) comparator._compare() diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_compare_event.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_compare_event.py index 771e1993398704774b9cb8a5b48c350f0a73b5bd..8b2c40dcd4a05a7966816e5ab1e50da84fcc34ac 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_compare_event.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_compare_event.py @@ -1,7 +1,10 @@ import unittest -from compare_backend.compare_bean.origin_data_bean.compare_event import KernelEvent, MemoryEvent -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.compare_event import ( + KernelEvent, + MemoryEvent +) +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean class TestKernelEvent(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py index 869ee85570febe1d5db7c1a5aa6e89ac8392078d..94ff68eceb9b4682e67df455185fde634ea12b36 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py @@ -1,6 +1,7 @@ import unittest -from compare_backend.compare_bean.origin_data_bean.kernel_details_bean import KernelDetailsBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.kernel_details_bean \ + import KernelDetailsBean class TestKernelDetailsBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_memory_record_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_memory_record_bean.py index 3ec34ffbaa53d0369716e4df6a5633bae7eb28c1..2488105bec8cb8b41b14c7b21f4ab0e7d6742cb0 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_memory_record_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_memory_record_bean.py @@ -1,6 +1,7 @@ import unittest -from compare_backend.compare_bean.origin_data_bean.memory_record_bean import MemoryRecordBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.memory_record_bean \ + import MemoryRecordBean class TestMemoryRecordBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_operator_memory_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_operator_memory_bean.py index 027b620e81fc6c9fa5d4694258f69cccd21a7c91..750da0b9be8f4a0d0da918b8f46577c336f005d8 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_operator_memory_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_operator_memory_bean.py @@ -1,6 +1,7 @@ import unittest -from compare_backend.compare_bean.origin_data_bean.operator_memory_bean import OperatorMemoryBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.operator_memory_bean \ + import OperatorMemoryBean class TestOperatorMemoryBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_trace_event_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_trace_event_bean.py index 2bd56e611dcb4864335631c5cbd29c3fa8c8c8a8..8dbb9ac5d4f586953fc891fa005c10ddf86b6c43 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_trace_event_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/origin_data_bean/test_trace_event_bean.py @@ -1,6 +1,6 @@ import unittest -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean class TestTraceEventBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_communication_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_communication_bean.py index 2b5ee568287b47a9c8ee4b4f7c5989b42883a4d9..4c14de0a43ce5b7f057887313e5c8c0dfe4530d6 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_communication_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_communication_bean.py @@ -1,6 +1,6 @@ import unittest -from compare_backend.compare_bean.communication_bean import CommunicationBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.communication_bean import CommunicationBean class TestCommunicationBean(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_compare_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_compare_bean.py index f275c2261af172c99b97a4cc4775a3942ec4a967..39efc1ac88d9cba9a4466edd660742b2c702419f 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_compare_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_compare_bean.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import patch -from compare_backend.compare_bean.memory_compare_bean import MemoryCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.memory_compare_bean import MemoryCompareBean class MockNode: @@ -22,18 +22,21 @@ class TestMemoryCompareBean(unittest.TestCase): def test_row_when_valid_data(self): result = [2, self.name, None, None, 'add', 8, self.name, None, None, 'add', 8, 0, 1.0] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", return_value=[MockMemory(8)]): + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", + return_value=[MockMemory(8)]): mem = MemoryCompareBean(1, MockNode(self.name), MockNode(self.name)) self.assertEqual(mem.row, result) def test_row_when_invalid_base_data(self): result = [2, None, None, None, "", 0, self.name, None, None, 'add', 8, 8, float("inf")] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", return_value=[MockMemory(8)]): + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", + return_value=[MockMemory(8)]): mem = MemoryCompareBean(1, None, MockNode(self.name)) self.assertEqual(mem.row, result) def test_row_when_invalid_comparison_data(self): result = [2, self.name, None, None, 'add', 8, None, None, None, '', 0, -8, 0] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", return_value=[MockMemory(8)]): + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", + return_value=[MockMemory(8)]): mem = MemoryCompareBean(1, MockNode(self.name), None) self.assertEqual(mem.row, result) diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_statistic_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_statistic_bean.py index 40bed2160baa6ee2136791f6fb9d26f0bb0c30bc..48a4f0096b62c7f5c015c342c8af61997d8cf4ef 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_statistic_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_memory_statistic_bean.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import patch -from compare_backend.compare_bean.memory_statistic_bean import MemoryStatisticBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.memory_statistic_bean import MemoryStatisticBean class MockMemory: @@ -15,21 +15,21 @@ class TestMemoryStatisticBean(unittest.TestCase): def test_row_when_valid_data(self): result = [None, self.name, 8.0, 40.0, 2, 4.0, 20.0, 1, -20.0, 0.5] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", return_value=[MockMemory(10240, 2000), MockMemory(10240, 2000)]): bean = MemoryStatisticBean(self.name, [1, 1], [1]) self.assertEqual(bean.row, result) def test_row_when_invalid_base_data(self): result = [None, self.name, 0, 0, 0, 4.0, 20.0, 1, 20.0, float("inf")] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", return_value=[MockMemory(10240, 2000), MockMemory(10240, 2000)]): bean = MemoryStatisticBean(self.name, [], [1]) self.assertEqual(bean.row, result) def test_row_when_invalid_comparison_data(self): result = [None, self.name, 8.0, 40.0, 2, 0, 0, 0, -40.0, 0] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_memory", return_value=[MockMemory(10240, 2000), MockMemory(10240, 2000)]): bean = MemoryStatisticBean(self.name, [1, 1], []) self.assertEqual(bean.row, result) diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_compare_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_compare_bean.py index ab8dec156f9589ccf516cda2c7ecd3ecf854f963..15099a5f54208b6c3e31147b1d04578ea0c29f84 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_compare_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_compare_bean.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import patch -from compare_backend.compare_bean.operator_compare_bean import OperatorCompareBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.operator_compare_bean import OperatorCompareBean class MockNode: @@ -22,18 +22,21 @@ class TestOperatorCompareBean(unittest.TestCase): def test_row_when_valid_data(self): result = [2, self.name, None, None, 'add', 8, self.name, None, None, 'add', 8, 0, 1.0] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", return_value=[MockKernel(8)]): + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", + return_value=[MockKernel(8)]): op = OperatorCompareBean(1, MockNode(self.name), MockNode(self.name)) self.assertEqual(op.row, result) def test_row_when_invalid_base_data(self): result = [2, None, None, None, "", 0, self.name, None, None, 'add', 8, 8, float("inf")] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", return_value=[MockKernel(8)]): + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", + return_value=[MockKernel(8)]): op = OperatorCompareBean(1, None, MockNode(self.name)) self.assertEqual(op.row, result) def test_row_when_invalid_comparison_data(self): result = [2, self.name, None, None, 'add', 8, None, None, None, '', 0, -8, 0] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", return_value=[MockKernel(8)]): + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", + return_value=[MockKernel(8)]): op = OperatorCompareBean(1, MockNode(self.name), None) self.assertEqual(op.row, result) diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_statistic_bean.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_statistic_bean.py index 4c4e338ce593a588316d2461eac03fd4e9677233..5293d428468678add6c7829888a69d6a2e409260 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_statistic_bean.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_operator_statistic_bean.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import patch -from compare_backend.compare_bean.operator_statistic_bean import OperatorStatisticBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.operator_statistic_bean import OperatorStatisticBean class MockKernel: @@ -14,21 +14,21 @@ class TestOperatorStatisticBean(unittest.TestCase): def test_row_when_valid_data(self): result = [None, self.name, 8.0, 2, 4.0, 1, -4.0, 0.5] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", return_value=[MockKernel(2000), MockKernel(2000)]): bean = OperatorStatisticBean(self.name, [1, 1], [1]) self.assertEqual(bean.row, result) def test_row_when_invalid_base_data(self): result = [None, self.name, 0, 0, 4.0, 1, 4.0, float("inf")] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", return_value=[MockKernel(2000), MockKernel(2000)]): bean = OperatorStatisticBean(self.name, [], [1]) self.assertEqual(bean.row, result) def test_row_when_invalid_comparison_data(self): result = [None, self.name, 8.0, 2, 0, 0, -8.0, 0] - with patch("compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", + with patch("msprof_analyze.compare_tools.compare_backend.utils.tree_builder.TreeBuilder.get_total_kernels", return_value=[MockKernel(2000), MockKernel(2000)]): bean = OperatorStatisticBean(self.name, [1, 1], []) self.assertEqual(bean.row, result) diff --git a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_profiling_info.py b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_profiling_info.py index 59525f18f96236a7e0383d08721629318f690f1b..b408f734a3c575c4e6dd25f4a826b4897c7acf04 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_profiling_info.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/compare_bean/test_profiling_info.py @@ -1,6 +1,6 @@ import unittest -from compare_backend.compare_bean.profiling_info import ProfilingInfo +from msprof_analyze.compare_tools.compare_backend.compare_bean.profiling_info import ProfilingInfo class TestProfilingInfo(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py index c4c26bd6ddee33f9fcacf11a27728f571b2fe435..de2ef0a46800bade04cf955cee68ce6f82acd274 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py @@ -1,8 +1,11 @@ import unittest from unittest.mock import patch -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.profiling_parser.base_profiling_parser import BaseProfilingParser, ProfilingResult +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import ( + BaseProfilingParser, + ProfilingResult +) class ProfilingParser(BaseProfilingParser): @@ -100,55 +103,63 @@ class TestBaseProfilingParser(unittest.TestCase): def test_picking_torch_op_event(self): event = MockEvent(1, 2, 3) - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init({}, {}) self.assertTrue(parser._picking_torch_op_event(event)) def test_picking_kernel_event(self): event = MockEvent(1, 2, 3) - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init({}, {}) self.assertTrue(parser._picking_kernel_event(event)) def test_picking_flow_event(self): events = [MockEvent(1, 2, 3, "s"), MockEvent(1, 2, 3, "f")] - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init({}, {}) for event in events: self.assertTrue(parser._picking_flow_event(event)) def test_update_kernel_dict_when_valid_input(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init(self.flow_dict, self.all_kernels) parser._update_kernel_dict() self.assertEqual(len(parser._result_data.kernel_dict.get(12)), 2) def test_update_kernel_dict_when_without_kernels_return_null(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init(self.flow_dict, {}) parser._update_kernel_dict() self.assertEqual(len(parser._result_data.kernel_dict), 0) def test_update_kernel_dict_when_without_flow_return_null(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init({}, self.all_kernels) parser._update_kernel_dict() self.assertEqual(len(parser._result_data.kernel_dict), 0) def test_check_result_data(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init(self.flow_dict, self.all_kernels) parser._check_result_data() def test_load_data_when_valid_input(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init(self.flow_dict, self.all_kernels) result_data = parser.load_data() @@ -156,7 +167,8 @@ class TestBaseProfilingParser(unittest.TestCase): def test_update_communication_dict(self): result = {'allreduce': {'comm_list': [2.0], 'comm_task': {'notify_wait': [1.0]}}} - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"): + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"): parser = ProfilingParser() parser.init({}, {}) parser._comm_task_list = [TraceEventBean(event) for event in self.task_events] diff --git a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py index 59ca545e2d09634443e37aca6dded79920878828..9e4d76bf97823bde34987a6bfb9e385fde6f581f 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py @@ -2,9 +2,9 @@ import unittest from collections import defaultdict from unittest.mock import patch -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.profiling_parser.base_profiling_parser import ProfilingResult -from compare_backend.profiling_parser.gpu_profiling_parser import GPUProfilingParser +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import ProfilingResult +from msprof_analyze.compare_tools.compare_backend.profiling_parser.gpu_profiling_parser import GPUProfilingParser class TestGpuProfilingParser(unittest.TestCase): @@ -52,8 +52,10 @@ class TestGpuProfilingParser(unittest.TestCase): other_event = {"ph": "X", "name": "other", "pid": 1, "tid": 1, "ts": 6, "dur": 1} def test_update_memory_list_when_valid_input(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.gpu_profiling_parser.GPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "gpu_profiling_parser.GPUProfilingParser.__init__", return_value=None): res = GPUProfilingParser({}, {}) res._enable_memory_compare = True @@ -64,8 +66,10 @@ class TestGpuProfilingParser(unittest.TestCase): self.assertEqual(res._result_data.memory_list[0].memory_details, ", (1, 2), [duration: 1.0], [size: 0.5]\n") def test_calculate_performance_time_when_valid_input(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.gpu_profiling_parser.GPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "gpu_profiling_parser.GPUProfilingParser.__init__", return_value=None): res = GPUProfilingParser({}, {}) res._profiling_type = "GPU" @@ -90,8 +94,10 @@ class TestGpuProfilingParser(unittest.TestCase): self.assertEqual(res._result_data.overall_metrics.compute_time, 7) def test_picking_memory_event_when_valid_input(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.gpu_profiling_parser.GPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "gpu_profiling_parser.GPUProfilingParser.__init__", return_value=None): res = GPUProfilingParser({}, {}) res._memory_events = [] @@ -102,8 +108,10 @@ class TestGpuProfilingParser(unittest.TestCase): def test_is_torch_op_event_when_valid_input(self): event_list = [{"cat": "cpu_op"}, {"cat": "user_annotation"}, {"cat": "cuda_runtime"}, {"cat": "operator"}] - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.gpu_profiling_parser.GPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "gpu_profiling_parser.GPUProfilingParser.__init__", return_value=None): res = GPUProfilingParser({}, {}) for event in event_list: @@ -115,8 +123,10 @@ class TestGpuProfilingParser(unittest.TestCase): def test_is_kernel_event_when_valid_input(self): event_list1 = [{"cat": "kernel", "name": "matmul"}, {"cat": "kernel", "name": "nccl_reduce"}] event_list2 = [{"cat": "async", "name": "nccl_reduce"}, {"cat": "cpu_op", "name": "aten::to"}] - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.gpu_profiling_parser.GPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "gpu_profiling_parser.GPUProfilingParser.__init__", return_value=None): res = GPUProfilingParser({}, {}) for event in event_list1: @@ -127,8 +137,10 @@ class TestGpuProfilingParser(unittest.TestCase): self.assertFalse(result) def test_is_flow_event_when_valid_input(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.gpu_profiling_parser.GPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "gpu_profiling_parser.GPUProfilingParser.__init__", return_value=None): res = GPUProfilingParser({}, {}) res._flow_cat = ("async_gpu",) diff --git a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py index a835dd9c1234120d3d19e701a2a1250034c47cd7..8f33065c16f2d7a00ab08cec6305c0f237a66252 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py @@ -1,10 +1,11 @@ import unittest from unittest.mock import patch -from compare_backend.compare_bean.origin_data_bean.operator_memory_bean import OperatorMemoryBean -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.profiling_parser.base_profiling_parser import ProfilingResult -from compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingParser +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.operator_memory_bean \ + import OperatorMemoryBean +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.profiling_parser.base_profiling_parser import ProfilingResult +from msprof_analyze.compare_tools.compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingParser class TestNPUProfilingParser(unittest.TestCase): @@ -21,8 +22,10 @@ class TestNPUProfilingParser(unittest.TestCase): {"ph": "M", "name": "thread_sort_index", "pid": 7, "tid": 3, "args": {"sort_index": 0}}] def test_update_memory_list_when_invalid_path(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None): res = NPUProfilingParser({}, {}) res._operator_memory_path = "" @@ -36,10 +39,13 @@ class TestNPUProfilingParser(unittest.TestCase): OperatorMemoryBean({"Name": "cann::add", "Size(KB)": 512, "Allocation Time(us)": 2, "Release Time(us)": 4}), OperatorMemoryBean( {"Name": "aten::add", "Size(KB)": 512, "Allocation Time(us)": 7, "Release Time(us)": 10})] - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None), \ - patch("msprof_analyze.prof_common.file_manager.FileManager.read_csv_file", return_value=memory_data): + patch("msprof_analyze.prof_common.file_manager.FileManager.read_csv_file", + return_value=memory_data): res = NPUProfilingParser({}, {}) res._path_level='' res._operator_memory_path = "" @@ -52,8 +58,10 @@ class TestNPUProfilingParser(unittest.TestCase): self.assertEqual(res._result_data.memory_list[0].duration, 2) def test_picking_hccl_event(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None): res = NPUProfilingParser({}, {}) res._hccl_pid = 7 @@ -67,8 +75,10 @@ class TestNPUProfilingParser(unittest.TestCase): self.assertEqual(len(res._comm_list), 1) def test_picking_task_queue_data(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None): res = NPUProfilingParser({}, {}) res._enqueue_dict = {} @@ -83,8 +93,10 @@ class TestNPUProfilingParser(unittest.TestCase): self.assertEqual(len(res._dequeue_data), 1) def test_picking_overlap_analysis_data(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None): res = NPUProfilingParser({}, {}) res._overlap_analysis = [] @@ -97,8 +109,10 @@ class TestNPUProfilingParser(unittest.TestCase): self.assertFalse(result) def test_is_kernel_event(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None): res = NPUProfilingParser({}, {}) res._kernel_pid = 5 @@ -107,27 +121,34 @@ class TestNPUProfilingParser(unittest.TestCase): self.assertFalse(res._is_kernel_event(TraceEventBean({"pid": 1, "ph": "x"}))) def test_is_flow_event(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None): res = NPUProfilingParser({}, {}) self.assertTrue(res._is_flow_event(TraceEventBean({"cat": "async_npu"}))) self.assertFalse(res._is_flow_event(TraceEventBean({"cat": "async"}))) def test_is_torch_op_event(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None): res = NPUProfilingParser({}, {}) self.assertTrue(res._is_torch_op_event(TraceEventBean({"cat": "cpu_op"}))) self.assertFalse(res._is_torch_op_event(TraceEventBean({"cat": "async"}))) def test_filter_meta_id(self): - with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ - patch("compare_backend.profiling_parser.npu_profiling_parser.NPUProfilingParser.__init__", + with patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "base_profiling_parser.BaseProfilingParser.__init__"), \ + patch("msprof_analyze.compare_tools.compare_backend.profiling_parser." + "npu_profiling_parser.NPUProfilingParser.__init__", return_value=None), \ patch( - "compare_backend.profiling_parser.npu_profiling_parser.BaseProfilingParser._trace_event_generator", + "compare_backend.profiling_parser.npu_profiling_parser.BaseProfilingParser." + "_trace_event_generator", return_value=(TraceEventBean(event) for event in self.meta_events)): res = NPUProfilingParser({}, {}) res._hccl_op_tid_list = [] diff --git a/profiler/msprof_analyze/test/ut/compare_tools/utils/test_name_function.py b/profiler/msprof_analyze/test/ut/compare_tools/utils/test_name_function.py index 2903f9838bc1bf093ac78238076c56756e77cf07..99b90051a456a0f531ae074cd19b55e53188f70b 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/utils/test_name_function.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/utils/test_name_function.py @@ -1,8 +1,8 @@ import unittest -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.utils.name_function import NameFunction -from compare_backend.utils.torch_op_node import TorchOpNode +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.utils.name_function import NameFunction +from msprof_analyze.compare_tools.compare_backend.utils.torch_op_node import TorchOpNode class Args: diff --git a/profiler/msprof_analyze/test/ut/compare_tools/utils/test_tree_builder.py b/profiler/msprof_analyze/test/ut/compare_tools/utils/test_tree_builder.py index 326a424d3dd9a36d158816ba73ffcf260ac583d9..389d11c5f3a0f0455e5bcdd7ca94fd6bb4c1cb4f 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/utils/test_tree_builder.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/utils/test_tree_builder.py @@ -1,8 +1,8 @@ import unittest -from compare_backend.compare_bean.origin_data_bean.compare_event import MemoryEvent -from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean -from compare_backend.utils.tree_builder import TreeBuilder +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.compare_event import MemoryEvent +from msprof_analyze.compare_tools.compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean +from msprof_analyze.compare_tools.compare_backend.utils.tree_builder import TreeBuilder class TestUtils(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/view/test_excel_view.py b/profiler/msprof_analyze/test/ut/compare_tools/view/test_excel_view.py index aa500c4242d567f3f588c7702202570f5f17d276..20357a886b52d4adae709513f6c78a958f447530 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/view/test_excel_view.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/view/test_excel_view.py @@ -2,7 +2,7 @@ import os import unittest from unittest.mock import patch -from compare_backend.view.excel_view import ExcelView +from msprof_analyze.compare_tools.compare_backend.view.excel_view import ExcelView class TestExcelView(unittest.TestCase): @@ -14,5 +14,6 @@ class TestExcelView(unittest.TestCase): os.remove(self.file_path) def test_generate_view(self): - with patch("compare_backend.view.work_sheet_creator.WorkSheetCreator.create_sheet"): + with patch("msprof_analyze.compare_tools.compare_backend.view.work_sheet_creator." + "WorkSheetCreator.create_sheet"): ExcelView({"table1": {}, "table2": {}}, self.file_path, {}).generate_view() diff --git a/profiler/msprof_analyze/test/ut/compare_tools/view/test_screen_view.py b/profiler/msprof_analyze/test/ut/compare_tools/view/test_screen_view.py index caa25e396e4dae801305cc39f78e657e1bc9601a..027cebbb9ca3384cd4072afd40cfd667488d37d7 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/view/test_screen_view.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/view/test_screen_view.py @@ -1,6 +1,6 @@ import unittest -from compare_backend.view.screen_view import ScreenView +from msprof_analyze.compare_tools.compare_backend.view.screen_view import ScreenView class TestScreenView(unittest.TestCase): diff --git a/profiler/msprof_analyze/test/ut/compare_tools/view/test_worker_sheet_creator.py b/profiler/msprof_analyze/test/ut/compare_tools/view/test_worker_sheet_creator.py index 1e80931ff5d014adf57012e22e5850e8c75af68c..587cccca986f88769eef18f38162ea92eaa09871 100644 --- a/profiler/msprof_analyze/test/ut/compare_tools/view/test_worker_sheet_creator.py +++ b/profiler/msprof_analyze/test/ut/compare_tools/view/test_worker_sheet_creator.py @@ -4,8 +4,8 @@ import unittest import pandas as pd from xlsxwriter import Workbook -from compare_backend.utils.excel_config import ExcelConfig -from compare_backend.view.work_sheet_creator import WorkSheetCreator +from msprof_analyze.compare_tools.compare_backend.utils.excel_config import ExcelConfig +from msprof_analyze.compare_tools.compare_backend.view.work_sheet_creator import WorkSheetCreator class TestWorkerSheetCreator(unittest.TestCase): diff --git "a/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" "b/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" index 67064cd045c9b65da7bbf9bbce5378cb47dabcad..c78d206c1a47d0e39555574ac78b111cc0d37c53 100644 --- "a/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" +++ "b/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" @@ -2,13 +2,13 @@ | 软件类型 | 软件名 | 路径 | 类型 | 内容 | 用途说明 | |------|----------------------------------------------------|------------------------------------------|------|------------------------------------------------------------------------------------------------------------|--------------------| -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md" | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0067.html | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/devtools/auxiliarydevtool/aoe_16_045.html | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://www.mindspore.cn/lite/docs/en/master/use/cloud_infer/converter_tool_ascend.html#aoe-auto-tuning | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0059.html | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/config/config.ini | 公网地址 | https://gitee.com/ascend/mstt/tree/master/profiler | msprof-analyze工具地址 | -| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/LICENSE | 公网地址 | http://www.apache.org/licenses/LICENSE-2.0 | 开源软件协议地址 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/rules/aicpu_rules.ymal | 公网地址 | https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20AI%20CPU%20Operator%20Replacement.md | AI CPU 算子替换样例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/rules/environment_variable_info.yaml | 公网地址 | https://support.huawei.com/enterprise/zh/doc/EDOC1100371278/5eeeed85?idPath=23710424 | 组网指南 | -| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/config/config.ini | 公网地址 | pmail_mindstudio@huawei.com | 公网邮箱 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/config/config.ini | 公网地址 | https://gitee.com/ascend/mstt/blob/master/profiler/msprof_analyze/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md" | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0067.html | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/devtools/auxiliarydevtool/aoe_16_045.html | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/config/config.ini | 公网地址 | https://www.mindspore.cn/lite/docs/en/master/use/cloud_infer/converter_tool_ascend.html#aoe-auto-tuning | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0059.html | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/msprof_analyze/config/config.ini | 公网地址 | https://gitee.com/ascend/mstt/tree/master/profiler/msprof_analyze | msprof-analyze工具地址 | +| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/msprof_analyze/LICENSE | 公网地址 | http://www.apache.org/licenses/LICENSE-2.0 | 开源软件协议地址 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/rules/aicpu_rules.ymal | 公网地址 | https://gitee.com/ascend/mstt/blob/master/profiler/msprof_analyze/advisor/doc/Samples%20of%20AI%20CPU%20Operator%20Replacement.md | AI CPU 算子替换样例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/msprof_analyze/advisor/rules/environment_variable_info.yaml | 公网地址 | https://support.huawei.com/enterprise/zh/doc/EDOC1100371278/5eeeed85?idPath=23710424 | 组网指南 | +| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/msprof_analyze/config/config.ini | 公网地址 | pmail_mindstudio@huawei.com | 公网邮箱 |