diff --git a/profiler/advisor/advisor_backend/compute_advice/compute_advice_base.py b/profiler/advisor/advisor_backend/compute_advice/compute_advice_base.py index 811cce84e0d787c1fc71365a73eef0e560f176c4..cca97c1e77739c08d11dbaa9e03494936d16b382 100644 --- a/profiler/advisor/advisor_backend/compute_advice/compute_advice_base.py +++ b/profiler/advisor/advisor_backend/compute_advice/compute_advice_base.py @@ -42,7 +42,8 @@ class ComputeAdviceBase(AdviceBase): if not os.path.exists(self.collection_path): print("[ERROR] Path: {} is not exist.".format(self.collection_path)) return False - if os.path.isdir(self.collection_path) and self.collection_path.endswith("ascend_pt"): + if os.path.isdir(self.collection_path) and \ + (self.collection_path.endswith("ascend_pt") or self.collection_path.endswith("ascend_ms")): self.kernel_details_path = os.path.join(self.collection_path, "ASCEND_PROFILER_OUTPUT", "kernel_details.csv") if not os.path.exists(self.kernel_details_path): diff --git a/profiler/advisor/advisor_backend/timeline_advice/timeline_advice_base.py b/profiler/advisor/advisor_backend/timeline_advice/timeline_advice_base.py index e2aa1e5ae12bf9c03f0e3f163221df08fb20c4a4..0bcf9a8f68787fc42d1ed5f9e34eafd42e5cbb21 100644 --- a/profiler/advisor/advisor_backend/timeline_advice/timeline_advice_base.py +++ b/profiler/advisor/advisor_backend/timeline_advice/timeline_advice_base.py @@ -55,7 +55,8 @@ class TimelineAdviceBase(AdviceBase): if not os.path.exists(self.collection_path): logger.error("Path: %s is not exist.",str(self.collection_path)) return False - if os.path.isdir(self.collection_path) and self.collection_path.endswith("ascend_pt"): + if os.path.isdir(self.collection_path) and \ + (self.collection_path.endswith("ascend_pt") or self.collection_path.endswith("ascend_ms")): self.trace_view_path = os.path.join(self.collection_path, "ASCEND_PROFILER_OUTPUT", "trace_view.json") if not os.path.exists(self.trace_view_path): logger.error("trace_view.json is not exist in the Path: %s."\ diff --git a/profiler/advisor/analyzer/analyzer_controller.py b/profiler/advisor/analyzer/analyzer_controller.py index 4da80e08f46651664818ff90769412175b5a7ee4..d47d943415c28a149ed8e18936e91d80bb13260c 100644 --- a/profiler/advisor/analyzer/analyzer_controller.py +++ b/profiler/advisor/analyzer/analyzer_controller.py @@ -39,6 +39,7 @@ from profiler.advisor.common.enum_params_parser import EnumParamsParser from profiler.advisor.utils.utils import Timer, safe_index_value, safe_division, safe_index, convert_to_int from profiler.advisor.interface.interface import Interface from profiler.cluster_analyse.cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor +from profiler.cluster_analyse.cluster_data_preprocess.mindspore_data_preprocessor import MindsporeDataPreprocessor from profiler.prof_common.path_manager import PathManager from profiler.prof_common.constant import Constant @@ -185,28 +186,6 @@ class AnalyzerController: return True - @staticmethod - def _whether_include_mindspore_prof(profiling_path): - # 暂不支持Mindspore数据，支持后可删除该限制 - ASCEND_MS = "ascend_ms" - - has_ascend_ms_dirs = False - for root, dirs, _ in os.walk(profiling_path): - if root.endswith(ASCEND_MS): - has_ascend_ms_dirs = True - break - for dir_name in dirs: - if dir_name.endswith(ASCEND_MS): - has_ascend_ms_dirs = True - break - if has_ascend_ms_dirs: - break - - if has_ascend_ms_dirs: - logger.error("Advisor does not support data from MindSpore now, existing dirs end with 'ascend_ms'") - return True - - return False @staticmethod def _get_step_rank_for_cluster_statistic_diff(target_cluster_statistic_data, benchmark_cluster_statistic_data, @@ -318,7 +297,8 @@ class AnalyzerController: dimensions: analysis dimension, normally set as Interface.all_dimension, support specific dimension analysis such as ['computation'] or ['computation', 'schedule'] cann_version: cann version of your runtime, inpact on the analysis of affinity api and AICPU operators - torch_version: torch version of your runtime, inpact on the analysis of affinity api + profiling_type: profiling type of your runtime + profiling_version: profiling version of your runtime, inpact on the analysis of affinity api analysis_dimensions: can overwite dimensions. advisor_analyze_processes: number of processes to use while the training params pipeline parallel(pp) >1, can reduce the time of analysis. @@ -646,14 +626,6 @@ class AnalyzerController: logger.error(error_msg) return - # 暂不支持Mindspore数据，支持后可删除该限制 - if self._whether_include_mindspore_prof(profiling_path): - error_msg = f"Got *_ascend_ms dirs from {profiling_path}, skip analysis" - self._update_analysis_process_resp(pid, async_resp, error_msg=error_msg, - status_code=AsyncAnalysisStatus.FAILED_STATUS_CODE, - status=AsyncAnalysisStatus.FAILED) - logger.error(error_msg) - return if benchmark_profiling_path and not self._check_profiling_path_valid(benchmark_profiling_path): error_msg = (f"Got invalid argument '-bp/--benchmark_profiling_path' {benchmark_profiling_path}, " @@ -840,7 +812,16 @@ class AnalyzerController: return False path_list = [os.path.join(profiling_path, dir_name) for dir_name in os.listdir(profiling_path)] ascend_pt_dirs = [path for path in path_list if os.path.isdir(path) and path.endswith("ascend_pt")] - data_processor = PytorchDataPreprocessor(ascend_pt_dirs) + ascend_ms_dirs = [path for path in path_list if os.path.isdir(path) and path.endswith("ascend_ms")] + if ascend_ms_dirs and ascend_pt_dirs: + logger.error("Cannot analyze pytorch and mindspore meantime.") + return False + if not ascend_pt_dirs and not ascend_ms_dirs: + return False + if ascend_ms_dirs and not ascend_pt_dirs: + data_processor = MindsporeDataPreprocessor(ascend_ms_dirs) + elif ascend_pt_dirs and not ascend_ms_dirs: + data_processor = PytorchDataPreprocessor(ascend_pt_dirs) self.cluster_local_data_map[profiling_path] = data_processor.get_data_map() diff --git a/profiler/advisor/analyzer/base_analyzer.py b/profiler/advisor/analyzer/base_analyzer.py index b7fd46df44dd6053068b3a4fb7cb9acf0e602b9e..9a15f89547388302a08a36abfbf7729cc3bd9161 100644 --- a/profiler/advisor/analyzer/base_analyzer.py +++ b/profiler/advisor/analyzer/base_analyzer.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import os from functools import wraps from typing import Dict, List, Union from abc import abstractmethod, ABCMeta @@ -25,9 +26,16 @@ from profiler.advisor.result.result import OptimizeResult from profiler.advisor.display.html.render import HTMLRender from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.utils.utils import safe_division +from profiler.prof_common.file_manager import FileManager logger = logging.getLogger() +ASCEND_PT = "ascend_pt" +ASCEND_MS = "ascend_ms" +PROFILER_INFO_HEAD = "profiler_info_" +PROFILER_INFO_EXTENSION = ".json" +MS_VERSION = "ms_version" + class BaseAnalyzer(VersionControl, metaclass=ABCMeta): _SUPPORT_VERSIONS = EnumParamsParser().get_options(Constant.CANN_VERSION) @@ -38,11 +46,13 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): def __init__(self, collection_path, n_processes: int = 1, **kwargs): self.n_processes = n_processes + self.kwargs = kwargs + self.collection_path = collection_path self.cann_version = kwargs.get(Constant.CANN_VERSION, EnumParamsParser().get_default(Constant.CANN_VERSION)) - self.torch_version = kwargs.get(Constant.TORCH_VERSION, EnumParamsParser().get_default(Constant.TORCH_VERSION)) + self.profiling_type = self.identify_profiling_type( + EnumParamsParser().get_options(Constant.PROFILING_TYPE_UNDER_LINE)) + self.profiling_version = self.identify_profiling_version() self.html_render = HTMLRender() - self.collection_path = collection_path - self.kwargs = kwargs self.dataset_list: Dict[str, List[Dataset]] = {} self.init_dataset_list() self.result = OptimizeResult() @@ -94,6 +104,67 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): def get_priority(self, max_mem_op_dur): pass + def identify_profiling_type(self, profiling_type_list): + profiling_type = None + if self.collection_path.endswith(ASCEND_MS): + profiling_type = [elem for elem in profiling_type_list if Constant.MINDSPORE in elem][0] + elif self.collection_path.endswith(ASCEND_PT): + profiling_type = [elem for elem in profiling_type_list if Constant.PYTORCH in elem][0] + else: + for _, dirs, __ in os.walk(self.collection_path): + is_found_type = False + for dir in dirs: + if dir.endswith(ASCEND_MS): + profiling_type = [elem for elem in profiling_type_list if Constant.MINDSPORE in elem][0] + is_found_type = True + break + elif dir.endswith(ASCEND_PT): + profiling_type = [elem for elem in profiling_type_list if Constant.PYTORCH in elem][0] + is_found_type = True + break + if is_found_type: + break + if self.kwargs.get(Constant.PROFILING_TYPE_UNDER_LINE) and self.kwargs.get( + Constant.PROFILING_TYPE_UNDER_LINE) != profiling_type: + logger.warning("%s The input profiling type %s is inconsistent with the actual profiling type %s.", + self.__class__.__name__, self.kwargs.get(Constant.PROFILING_TYPE_UNDER_LINE), profiling_type) + if not profiling_type: + logger.warning("Unknown profiling type, the default value is set pytorch.") + profiling_type = profiling_type_list[0] + return profiling_type + + def identify_profiling_version(self): + profiling_version = "" + if Constant.MINDSPORE in self.profiling_type: + ascend_dirs = [] + if self.collection_path.endswith(ASCEND_MS): + ascend_dirs.append(self.collection_path) + else: + for root, dirs, _ in os.walk(self.collection_path): + for dir in dirs: + if dir.endswith(ASCEND_MS): + ascend_dirs.append(os.path.join(root, dir)) + if ascend_dirs: + ascend_dir = ascend_dirs[0] + for file_name in os.listdir(ascend_dir): + if file_name.startswith(PROFILER_INFO_HEAD) and file_name.endswith(PROFILER_INFO_EXTENSION): + file_path = os.path.join(ascend_dir, file_name) + config = FileManager.read_json_file(file_path) + profiling_version = config.get(MS_VERSION, "") + break + if profiling_version and self.kwargs.get(Constant.MINDSPORE_VERSION): + if profiling_version != self.kwargs.get(Constant.MINDSPORE_VERSION): + logger.warning("%s The input version %s is inconsistent with the actual version %s.", + self.__class__.__name__, self.kwargs.get(Constant.MINDSPORE_VERSION), + profiling_version) + elif Constant.PYTORCH in self.profiling_type: + profiling_version = self.kwargs.get(Constant.TORCH_VERSION, + EnumParamsParser().get_default(Constant.TORCH_VERSION)) + if self.kwargs.get(Constant.TORCH_VERSION) and profiling_version != self.kwargs.get(Constant.TORCH_VERSION): + logger.warning("%s The input version %s is inconsistent with the actual version %s.", + self.__class__.__name__, self.kwargs.get(Constant.TORCH_VERSION), profiling_version) + return profiling_version + def init_dataset_list(self) -> None: dataset_cls_list = self.dataset_cls_list if len(dataset_cls_list) == 0: diff --git a/profiler/advisor/analyzer/cluster/slow_link_analyzer.py b/profiler/advisor/analyzer/cluster/slow_link_analyzer.py index 10459e719294724e4836d2d2ef531bfef4f561e5..a49ba83774601a7c2b1aba8db2f1aac225cd7774 100644 --- a/profiler/advisor/analyzer/cluster/slow_link_analyzer.py +++ b/profiler/advisor/analyzer/cluster/slow_link_analyzer.py @@ -143,7 +143,8 @@ class SlowLinkAnalyzer(BaseAnalyzer): template_dir="templates", template_name="cluster_analysis.html", cann_version=self.cann_version, - torch_version=self.torch_version, + profiling_type=self.profiling_type, + profiling_version=self.profiling_version, result=result_for_html) def get_global_step_rank(self, bindwidth_type): diff --git a/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py b/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py index 0d73a74eea47af8cca3ada9ddc093c0106cf9356..7b2311157ae7207b4415b6c386854e39e61ba03d 100644 --- a/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py +++ b/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py @@ -134,7 +134,8 @@ class SlowRankAnalyzer(BaseAnalyzer): template_dir="templates", template_name="cluster_analysis.html", cann_version=self.cann_version, - torch_version=self.torch_version, + profiling_type=self.profiling_type, + profiling_version=self.profiling_version, result=result_for_html) def get_global_step_rank(self, dimension): diff --git a/profiler/advisor/analyzer/comparison/comparison_checker.py b/profiler/advisor/analyzer/comparison/comparison_checker.py index dc12f9353bab9696320ca77b9c34788a538708e2..4fa5f082c52562826a4cc0d08d4f3b574604fe97 100644 --- a/profiler/advisor/analyzer/comparison/comparison_checker.py +++ b/profiler/advisor/analyzer/comparison/comparison_checker.py @@ -67,6 +67,9 @@ class ComparisonChecker: if compare_mode is None: return self.compare_mode = compare_mode + if ("Api" in compare_mode) and self.benchmark_profiling_path.endswith("ascend_ms"): + logger.warning("The current compare mode %s does not support Mindspore.", compare_mode) + return compare_interface = ComparisonInterface(self.profiling_path, self.benchmark_profiling_path, self.step, self.benchmark_step) result = compare_interface.compare(self.compare_mode) diff --git a/profiler/advisor/analyzer/computation/profiling_analyzer.py b/profiler/advisor/analyzer/computation/profiling_analyzer.py index 6d525f303cc8c5971bda8a11d16d638ef3dcf2c3..dab4c7d95f4f58ec4c9d139a7f230857930f219a 100644 --- a/profiler/advisor/analyzer/computation/profiling_analyzer.py +++ b/profiler/advisor/analyzer/computation/profiling_analyzer.py @@ -90,6 +90,9 @@ class ProfilingAnalyzer(BaseAnalyzer, ABC): class DynamicShapeAnalyzer(ProfilingAnalyzer): def __init__(self, collection_path, **kwargs) -> None: super().__init__(collection_path, **kwargs) + if collection_path.endswith("ascend_ms"): + logger.warning("Dynamic shape analyzer does not support Mindspore.") + return self.checker = DynamicShapeChecker(self.cann_version) diff --git a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py b/profiler/advisor/analyzer/overall/overall_summary_analyzer.py index 3df71fe8149dec81d40030d1520863be1d2723f8..143eb854be2f90e9b0325ace537cc3961b54d747 100644 --- a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py +++ b/profiler/advisor/analyzer/overall/overall_summary_analyzer.py @@ -230,7 +230,8 @@ class OverallSummaryAnalyzer(BaseAnalyzer): template_dir="templates", template_name="cluster_analysis.html", cann_version=self.cann_version, - torch_version=self.torch_version, + profiling_type=self.profiling_type, + profiling_version=self.profiling_version, result=result_for_html) def get_priority(self): diff --git a/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py b/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py index ed0500d3b23440151d60b8186e7e12aba007ab89..4f84d93a5699ba8eff33bdbce35ed9666241cd7d 100644 --- a/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py +++ b/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py @@ -49,6 +49,9 @@ class OpDispatchAnalyzer(BaseAnalyzer): :param data: input datasets :return: result """ + if "mindspore" in self.profiling_type: + logger.warning("The analyzer %s does not support MindSpore.", self.__class__.__name__) + return self.result self.get_op_compile_info(self.dataset) self.make_record(self.result) self.make_render(self.html_render, rank=kwargs.get('rank')) diff --git a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py b/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py index 80ac80061557242f71bb939ba2f17c5ca68e6862..098fab15316a9bf541c0e68e8d3838d52723d5a7 100644 --- a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py +++ b/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py @@ -56,7 +56,9 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): for mode in [Constant.ATEN.lower(), Constant.OPTIMIZER.lower()]: - for op_combined, npu_apis in tqdm(getattr(init_timeline_ops_db(self.cann_version, self.torch_version), + for op_combined, npu_apis in tqdm(getattr(init_timeline_ops_db(self.cann_version, + self.profiling_type, + self.profiling_version), f"_{mode}_op_api_map").items(), leave=False, ncols=100, desc="Scanning timeline for affinity apis"): for npu_api in npu_apis.split("/"): @@ -94,7 +96,7 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): return desc = f"Found {len(format_timeline_result(self.matched_op_stacks))} apis to be replaced" \ - f" based on the runtime env cann-{self.cann_version} and torch-{self.torch_version}" + f" based on the runtime env cann-{self.cann_version} and torch-{self.profiling_version}" suggestion = "Please replace training api according to sub table 'Affinity training api'" if self.empty_stacks: desc += ", but with no stack" @@ -131,7 +133,8 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): template_dir="templates", template_name="affinity_api.html", cann_version=self.cann_version, - torch_version=self.torch_version, + profiling_type=self.profiling_type, + profiling_version=self.profiling_version, empty_stacks=self.empty_stacks, with_stack_doc_url=Config().timeline_with_stack_doc_url, api_doc_url=Config().timeline_api_doc_url, diff --git a/profiler/advisor/analyzer/schedule/gc/gc_analyzer.py b/profiler/advisor/analyzer/schedule/gc/gc_analyzer.py index b59a8fc2e2a25428e34cb51917462f9f6162bc46..8c77e93b333322fd04cd239caa4d8c49a20b7cc6 100644 --- a/profiler/advisor/analyzer/schedule/gc/gc_analyzer.py +++ b/profiler/advisor/analyzer/schedule/gc/gc_analyzer.py @@ -36,6 +36,9 @@ class GcAnalyzer(BaseAnalyzer): @BaseAnalyzer.check_data((ScheduleAnalysisDataset.get_key(),)) def optimize(self, **kwargs): + if "mindspore" in self.profiling_type: + logger.warning("The analyzer %s does not support MindSpore.", self.__class__.__name__) + return self.result gc_checker = GcChecker() gc_checker.check_gc(self.timeline_event_dataset, rank=kwargs.get("rank"), stage=kwargs.get("stage")) gc_checker.make_record(self.result) diff --git a/profiler/advisor/common/timeline/fusion_ops_db.py b/profiler/advisor/common/timeline/fusion_ops_db.py index 171786b15c5637c899f01b1a7c2fe4c16de61e79..11727bf3ed299777a3dab3457c396bff3b75365e 100644 --- a/profiler/advisor/common/timeline/fusion_ops_db.py +++ b/profiler/advisor/common/timeline/fusion_ops_db.py @@ -29,10 +29,12 @@ logger = logging.getLogger() logger.setLevel(get_log_level()) -def init_timeline_ops_db(cann_version=None, torch_version=None): +def init_timeline_ops_db(cann_version=None, profiling_type=None, profiling_version=None): logger.debug("init operators database") - return FusionOperatorDB(cann_version=cann_version, torch_version=torch_version) + return FusionOperatorDB(cann_version=cann_version, + profiling_type=profiling_type, + profiling_version=profiling_version) def get_timeline_fusion_ops_yaml_path(): @@ -65,17 +67,18 @@ def get_timeline_fusion_ops_yaml_path(): class FusionOperatorDB: - def __init__(self, file_path=None, cann_version=None, torch_version=None): + def __init__(self, cann_version=None, profiling_type=None, profiling_version=None): self.timeline_fusion_ops_yaml_path = os.path.normpath(get_timeline_fusion_ops_yaml_path()) - self.cann_version = cann_version or EnumParamsParser().get_default(Constant.CANN_VERSION) - self.torch_version = torch_version or EnumParamsParser().get_default(Constant.TORCH_VERSION) + self.profiling_type = profiling_type or EnumParamsParser().get_default(Constant.PROFILING_TYPE_UNDER_LINE) + self.profiling_version = profiling_version or EnumParamsParser().get_default(Constant.PROFILING_TYPE_UNDER_LINE) self._supported_version_dict = {} self.is_empty = False self.timeline_op_rule_handler = TimelineOpRuleHandler() - self.fusion_operator = self._load_yaml(self.timeline_fusion_ops_yaml_path) + self.fusion_operator = self._load_yaml( + self.timeline_fusion_ops_yaml_path) if profiling_type == Constant.PYTORCH else {} self._dequeue_op_names = [] self._aten_op_names = [] @@ -126,7 +129,7 @@ class FusionOperatorDB: def regenerate_timeline_op_rule_with_version(self, cann_version=None, torch_version=None): cann_version = cann_version or self.cann_version - torch_version = torch_version or self.torch_version + torch_version = torch_version or self.profiling_version unique_id = self._get_unique_id_in_supported_version_dict(cann_version=cann_version, torch_version=torch_version) self.regenerate_timeline_op_rule_with_unique_id(unique_id) @@ -180,7 +183,7 @@ class FusionOperatorDB: if not is_version_supported: # 若规则库不支持当前版本, 则log警告信息 logger.warning("Unsupported versions: cann-%s and torch-%s, supported version list of ['cann', 'torch'] " - "is %s", self.cann_version, self.torch_version, self._supported_version_dict.values()) + "is %s", self.cann_version, self.profiling_version, self._supported_version_dict.values()) return is_version_supported def _is_version_supported_in_supported_version_dict(self, cann_version=None, torch_version=None): @@ -208,7 +211,7 @@ class FusionOperatorDB: torch_version_list = [torch_version_list] cann_version = cann_version or self.cann_version - torch_version = torch_version or self.torch_version + torch_version = torch_version or self.profiling_version if (cann_version in cann_version_list) and (torch_version in torch_version_list): return True diff --git a/profiler/advisor/config/enum_parameters.yaml b/profiler/advisor/config/enum_parameters.yaml index b1a0548d480b8722609df40ae8e9331b5d3d34bd..678fe72b43c7f5b2fd66b3f38c3114cc9793cd50 100644 --- a/profiler/advisor/config/enum_parameters.yaml +++ b/profiler/advisor/config/enum_parameters.yaml @@ -6,7 +6,9 @@ arguments: - 7.0.RC1 - 7.0.0 - 8.0.RC1 - default: 8.0.RC1 + - 8.0.RC2 + - 8.0.0 + default: 8.0.0 torch_version: type: str @@ -14,7 +16,12 @@ arguments: - 1.11.0 - 2.1.0 default: 2.1.0 - + mindspore_version: + type: str + options: + - 2.3.0 + - 2.4.0 + default: 2.4.0 analysis_dimensions: type: list options: @@ -28,10 +35,11 @@ arguments: profiling_type: type: str options: - - ascend_pytorch_profiler + - pytorch - mslite - msprof - default: ascend_pytorch_profiler + - mindspore + default: pytorch envs: ADVISOR_ANALYZE_PROCESSES: diff --git a/profiler/advisor/config/profiling_data_version_config.yaml b/profiler/advisor/config/profiling_data_version_config.yaml index ed064e287583b699379b5707a21be301164c2b74..1c82a8a6bcfbd7ae3d09cab4ca1852b875b2e64c 100644 --- a/profiler/advisor/config/profiling_data_version_config.yaml +++ b/profiler/advisor/config/profiling_data_version_config.yaml @@ -1,4 +1,28 @@ versions: + - version: 8.0.0 + dirs_pattern: + ASCEND_PROFILER_OUTPUT: [ op_summary, msprof ] + ^PROF_\d{6}_\d{17}_\w+$: + mindstudio_profiler_output: [ op_summary, msprof ] + class_attr: + op_summary: OpSummary + msprof: Msprof + file_attr: + msprof: [trace_view.json, '^msprof_\d{14}\.json$'] + op_summary: [ kernel_details.csv, '^op_summary_\d{14}\.csv$' ] + + - version: 8.0.RC2 + dirs_pattern: + ASCEND_PROFILER_OUTPUT: [ op_summary, msprof ] + ^PROF_\d{6}_\d{17}_\w+$: + mindstudio_profiler_output: [ op_summary, msprof ] + class_attr: + op_summary: OpSummary + msprof: Msprof + file_attr: + msprof: [trace_view.json, '^msprof_\d{14}\.json$'] + op_summary: [ kernel_details.csv, '^op_summary_\d{14}\.csv$' ] + - version: 8.0.RC1 dirs_pattern: ASCEND_PROFILER_OUTPUT: [ op_summary, msprof ] diff --git a/profiler/advisor/dataset/communication/communication_dataset.py b/profiler/advisor/dataset/communication/communication_dataset.py index 6a346f52ada4d53ac630d2aeecff6304a2a69b87..d11e18c3e575afe7bac0e4eeb795e615ecfe37b5 100644 --- a/profiler/advisor/dataset/communication/communication_dataset.py +++ b/profiler/advisor/dataset/communication/communication_dataset.py @@ -30,7 +30,7 @@ class CommunicationDataset: def __init__(self, collection_path, data: dict, **kwargs) -> None: self.timeline_dir = collection_path - if not self.timeline_dir.endswith("ascend_pt"): + if not self.timeline_dir.endswith("ascend_pt") and not self.timeline_dir.endswith("ascend_ms"): return self.timeline_data_list = self.get_file_path_from_directory( self.timeline_dir, diff --git a/profiler/advisor/display/html/templates/affinity_api.html b/profiler/advisor/display/html/templates/affinity_api.html index 7cd3d7ad33d0220c7aba055721eddf049161a0d8..b227afae9624cadb5ce753c80f1e8719040eacb4 100644 --- a/profiler/advisor/display/html/templates/affinity_api.html +++ b/profiler/advisor/display/html/templates/affinity_api.html @@ -8,14 +8,14 @@ The analysis results of following affinity APIs are based on runtime env cann-{{ cann_version }} and - torch-{{ torch_version }} + {{profiling_type}}-{{ profiling_type }}
{% if empty_stacks %} Suggestion: These APIs have no code stack. If parameter 'with_stack=False' was set while profiling, please refer to - Ascend PyTorch Profiler to set + Ascend Profiler to set 'with_stack=True'. Otherwise, ignore following affinity APIs due to backward broadcast lack of stack. {% endif %} diff --git a/profiler/advisor/rules/timeline_fusion_ops.yaml b/profiler/advisor/rules/timeline_fusion_ops.yaml index 46e02fef77785a3bf3da65899539c42ac05fadd4..3337c938625ccd4b4ea77a0dafa9879222cf1bfe 100644 --- a/profiler/advisor/rules/timeline_fusion_ops.yaml +++ b/profiler/advisor/rules/timeline_fusion_ops.yaml @@ -45,6 +45,18 @@ "(slice|chunk)-mul-mul-sigmoid" ] - cann_version: 8.0.RC1 + torch_version: [1.11.0, 2.1.0] + unique_id: 3 + inherit_unique_id: 2 + operator_rules: + aten: + add: + torch_npu.npu_geglu: [ "(slice|chunk)-gelu-mul", "(slice|chunk)-mul-gelu" ] + torch_npu.npu_group_norm_silu: [ "group_norm-silu" ] + torch.addmm: [ "mul-mul-add" ] + torch_npu.npu_add_layer_norm: [ "add-layer_norm" ] + +- cann_version: 8.0.0 torch_version: [1.11.0, 2.1.0] unique_id: 3 inherit_unique_id: 2 diff --git a/profiler/cli/analyze_cli.py b/profiler/cli/analyze_cli.py index c91c55c337a332bf631f62624f27da17a5e0e96c..7c93d224cdec1c8c574d152639ab4ed948c7a04b 100644 --- a/profiler/cli/analyze_cli.py +++ b/profiler/cli/analyze_cli.py @@ -37,17 +37,12 @@ def analyze_cli(**kwargs): default=EnumParamsParser().get_default(Constant.CANN_VERSION), help='The CANN software version, which can be viewed by executing the following command: ' '"cat /usr/local/Ascend/ascend-toolkit/latest/aarch64-linux/ascend_toolkit_install.info"') -@click.option('--torch_version', '-tv', 'torch_version', - type=click.Choice(EnumParamsParser().get_options(Constant.TORCH_VERSION), case_sensitive=False), - default=EnumParamsParser().get_default(Constant.TORCH_VERSION), - help='The runtime torch version, which can be detected by exec command "pip show torch"') @click.option("-pt", "--profiling_type", metavar="", - default=EnumParamsParser().get_default(Constant.PROFILING_TYPE_UNDER_LINE), required=False, type=click.Choice(EnumParamsParser().get_options(Constant.PROFILING_TYPE_UNDER_LINE)), - help="enter the profiling type, selectable range ascend_pytorch_profiler, mslite ,msprof") + help="enter the profiling type, selectable range pytorch, mindspore, mslite ,msprof") @click.option("--force", is_flag=True, help="Indicates whether to skip file size verification and owner verification") diff --git a/profiler/cluster_analyse/prof_bean/step_trace_time_bean.py b/profiler/cluster_analyse/prof_bean/step_trace_time_bean.py index b0a3be4f5eaccea70aa912bc85e68d70dbda3bde..3cf4bc4fa5b1a8c8ccf93c010fe5fe8e9843e513 100644 --- a/profiler/cluster_analyse/prof_bean/step_trace_time_bean.py +++ b/profiler/cluster_analyse/prof_bean/step_trace_time_bean.py @@ -14,6 +14,10 @@ # limitations under the License. +import logging + +logger = logging.getLogger() + class StepTraceTimeBean: STEP = "Step" COMPLEMENT_HEADER = ["Step", "Type", "Index"] @@ -27,7 +31,11 @@ class StepTraceTimeBean: for field_name in self._data.keys(): if field_name == self.STEP: continue - row.append(float(self._data.get(field_name, ))) + try: + row.append(float(self._data.get(field_name, ))) + except Exception as e: + logger.warning(e) + row.append(0) return row @property diff --git a/profiler/prof_common/constant.py b/profiler/prof_common/constant.py index 86c86a8fe527425d30a37e296f778ed6f4d087f6..78cf1070ffd8ab1233065c6a7e25a3df9e46f33d 100644 --- a/profiler/prof_common/constant.py +++ b/profiler/prof_common/constant.py @@ -379,4 +379,8 @@ class Constant(object): DISABLE_PROFILING_COMPARISON = "DISABLE_PROFILING_COMPARISON" FREE_DURATION_FOR_GC_ANALYSIS = "FREE_DURATION_FOR_GC_ANALYSIS" - DISABLE_AFFINITY_API = "DISABLE_AFFINITY_API" \ No newline at end of file + DISABLE_AFFINITY_API = "DISABLE_AFFINITY_API" + + MINDSPORE_VERSION = "mindspore_version" + PYTORCH = "pytorch" + MINDSPORE = "mindspore" \ No newline at end of file diff --git a/profiler/test/st/advisor/test_advisor_cmd_single_ascend_pt_no_compare.py b/profiler/test/st/advisor/test_advisor_cmd_single_ascend_pt_no_compare.py index ce6eeb3a39c6944037abeeb835bdad53f8e7efbd..dfb7250480592430770ac63ac3be4a1aec072c51 100644 --- a/profiler/test/st/advisor/test_advisor_cmd_single_ascend_pt_no_compare.py +++ b/profiler/test/st/advisor/test_advisor_cmd_single_ascend_pt_no_compare.py @@ -16,7 +16,7 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): "/home/dcs-50/smoke_project_for_msprof_analyze/mstt_profiler/st_data") BASE_PROFILING_PATH = os.path.join(ST_DATA_PATH, "cluster_data_3", "n122-122-067_12380_20240912033946038_ascend_pt") OUTPUT_PATH = os.path.join(os.path.abspath(os.path.dirname(__file__)), "TestAdvisorCmdSingleAscendPtNoCompare") - ALL_OUTPUT_PATH = os.path.join(OUTPUT_PATH,"all") + ALL_OUTPUT_PATH = os.path.join(OUTPUT_PATH, "all") COMPUTATION_OUTPUT_PATH = os.path.join(OUTPUT_PATH, "computation") SCHEDULE_OUTPUT_PATH = os.path.join(OUTPUT_PATH, "schedule") RESULT_EXCEL = {} @@ -27,22 +27,21 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): PathManager.make_dir_safety(self.ALL_OUTPUT_PATH) PathManager.make_dir_safety(self.COMPUTATION_OUTPUT_PATH) PathManager.make_dir_safety(self.SCHEDULE_OUTPUT_PATH) - cmd_all = ["msprof-analyze", "advisor", "all" ,"-d", self.BASE_PROFILING_PATH, "-o",self.ALL_OUTPUT_PATH] + cmd_all = ["msprof-analyze", "advisor", "all", "-d", self.BASE_PROFILING_PATH, "-o", self.ALL_OUTPUT_PATH] if execute_cmd(cmd_all) != self.COMMAND_SUCCESS or not os.path.exists(self.ALL_OUTPUT_PATH): self.assertTrue(False, msg="advisor [all] task failed.") - cmd_computation = ["msprof-analyze", "advisor", "computation" ,"-d", self.BASE_PROFILING_PATH, "-o", - self.COMPUTATION_OUTPUT_PATH] - completed_process_computation = subprocess.run(cmd_computation, capture_output=True, shell=False) - if completed_process_computation.returncode != self.COMMAND_SUCCESS or not os.path.exists(self.COMPUTATION_OUTPUT_PATH): + + cmd_computation = ["msprof-analyze", "advisor", "computation", "-d", self.BASE_PROFILING_PATH, "-o", + self.COMPUTATION_OUTPUT_PATH] + if execute_cmd(cmd_computation) != self.COMMAND_SUCCESS or not os.path.exists(self.COMPUTATION_OUTPUT_PATH): self.assertTrue(False, msg="advisor [computation] task failed.") - cmd_schedule = ["msprof-analyze", "advisor", "schedule" ,"-d", self.BASE_PROFILING_PATH, "-o", - self.SCHEDULE_OUTPUT_PATH] - completed_process_schedule = subprocess.run(cmd_schedule, capture_output=True, shell=False) - if completed_process_schedule.returncode != self.COMMAND_SUCCESS or not os.path.exists( - self.SCHEDULE_OUTPUT_PATH): + + cmd_schedule = ["msprof-analyze", "advisor", "schedule", "-d", self.BASE_PROFILING_PATH, "-o", + self.SCHEDULE_OUTPUT_PATH] + if execute_cmd(cmd_schedule) != self.COMMAND_SUCCESS or not os.path.exists(self.SCHEDULE_OUTPUT_PATH): self.assertTrue(False, msg="advisor [schedule] task failed.") - self.RESULT_HTML,self.RESULT_EXCEL = get_files(self.OUTPUT_PATH) + self.RESULT_HTML, self.RESULT_EXCEL = get_files(self.OUTPUT_PATH) def teardown_class(self): PathManager.remove_path_safety(self.OUTPUT_PATH) @@ -58,32 +57,32 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): "Operator dispatch" ] - #True presents the attr is nan - description_len = [6,1,3,2,1,1,1] - suggestion_len = [True,1,1,2,5,1,1] - problem_count = [True,True,True,2.0,1.0,True,True] - total_time = [True,True,True,57674709.54,True,True,True] - time_ratio = [True,True,True,0.0,True,True,True] - income = [True,True,True,True,True,True,True] - income_ratio = [True,True,True,True,True,True,True] + # True presents the attr is nan + description_len = [6, 1, 3, 2, 1, 1, 1] + suggestion_len = [True, 1, 1, 2, 5, 1, 1] + problem_count = [True, True, True, 2.0, 1.0, True, True] + total_time = [True, True, True, 57674709.54, True, True, True] + time_ratio = [True, True, True, 0.0, True, True, True] + income = [True, True, True, True, True, True, True] + income_ratio = [True, True, True, True, True, True, True] try: - df = pd.read_excel(self.RESULT_EXCEL.get("all",None), sheet_name='problems',header=0) + df = pd.read_excel(self.RESULT_EXCEL.get("all", None), sheet_name='problems', header=0) except FileNotFoundError: - logging.error("File %s not found.", self.RESULT_EXCEL.get("all",None)) + logging.error("File %s not found.", self.RESULT_EXCEL.get("all", None)) return for index, row in df.iterrows(): self.assertEqual(category[index], row["category"]) self.assertEqual(description_len[index], len(row["description"].split("\n"))) - self.assertEqual(suggestion_len[index], isinstance(row["suggestion"],float) or - len(row["suggestion"].split("\n"))) + self.assertEqual(suggestion_len[index], isinstance(row["suggestion"], float) or + len(row["suggestion"].split("\n"))) self.assertEqual(problem_count[index], (math.isnan(row["problem count"]) or row["problem count"])) self.assertEqual(total_time[index], (math.isnan(row["total_time(us)"]) or - round(row["total_time(us)"],2))) - self.assertEqual(time_ratio[index], (math.isnan(row["time ratio"]) or round(row["time ratio"],2))) - self.assertEqual(income[index], (math.isnan(row["income(us)"]) or round(row["income(us)"],2))) + round(row["total_time(us)"], 2))) + self.assertEqual(time_ratio[index], (math.isnan(row["time ratio"]) or round(row["time ratio"], 2))) + self.assertEqual(income[index], (math.isnan(row["income(us)"]) or round(row["income(us)"], 2))) self.assertEqual(income_ratio[index], (math.isnan(row["income ratio"]) or - round(row["income ratio"],2))) + round(row["income ratio"], 2))) def test_computation_problems(self): category = [ @@ -92,7 +91,7 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): "Dynamic shape operator", ] - #True presents the attr is nan + # True presents the attr is nan description_len = [6, 2, 1] suggestion_len = [True, 2, 5] problem_count = [True, 2.0, 1.0] @@ -101,15 +100,15 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): income = [True, True, True] income_ratio = [True, True, True] try: - df = pd.read_excel(self.RESULT_EXCEL.get("computation",None), sheet_name='problems', header=0) + df = pd.read_excel(self.RESULT_EXCEL.get("computation", None), sheet_name='problems', header=0) except FileNotFoundError: - logging.error("File %s not found.", self.RESULT_EXCEL.get("computation",None)) + logging.error("File %s not found.", self.RESULT_EXCEL.get("computation", None)) return for index, row in df.iterrows(): self.assertEqual(category[index], row["category"]) self.assertEqual(description_len[index], len(row["description"].split("\n"))) - self.assertEqual(suggestion_len[index], (isinstance(row["suggestion"],float) or + self.assertEqual(suggestion_len[index], (isinstance(row["suggestion"], float) or len(row["suggestion"].split("\n")))) self.assertEqual(problem_count[index], (math.isnan(row["problem count"]) or row["problem count"])) self.assertEqual(total_time[index], (math.isnan(row["total_time(us)"]) or @@ -126,7 +125,7 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): "Operator dispatch" ] - #True presents the attr is nan + # True presents the attr is nan description_len = [6, 1, 1] suggestion_len = [True, 1, 1] problem_count = [True, True, True] @@ -135,15 +134,15 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): income = [True, True, True] income_ratio = [True, True, True] try: - df = pd.read_excel(self.RESULT_EXCEL.get("schedule",None), sheet_name='problems', header=0) + df = pd.read_excel(self.RESULT_EXCEL.get("schedule", None), sheet_name='problems', header=0) except FileNotFoundError: - logging.error("File %s not found.", self.RESULT_EXCEL.get("schedule",None)) + logging.error("File %s not found.", self.RESULT_EXCEL.get("schedule", None)) return for index, row in df.iterrows(): self.assertEqual(category[index], row["category"]) self.assertEqual(description_len[index], len(row["description"].split("\n"))) - self.assertEqual(suggestion_len[index], (isinstance(row["suggestion"] ,float) or + self.assertEqual(suggestion_len[index], (isinstance(row["suggestion"], float) or len(row["suggestion"].split("\n")))) self.assertEqual(problem_count[index], (math.isnan(row["problem count"]) or row["problem count"])) self.assertEqual(total_time[index], (math.isnan(row["total_time(us)"]) or @@ -152,6 +151,7 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): self.assertEqual(income[index], (math.isnan(row["income(us)"]) or round(row["income(us)"], 2))) self.assertEqual(income_ratio[index], (math.isnan(row["income ratio"]) or round(row["income ratio"], 2))) + def test_overall_summary(self): performance_index = [ "Computing Time", " -- Flash Attention", @@ -170,9 +170,9 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): test_pattern = ["all", "computation", "schedule"] for pattern in test_pattern: try: - df = pd.read_excel(self.RESULT_EXCEL.get(pattern,None), sheet_name='overall summary', header=0) + df = pd.read_excel(self.RESULT_EXCEL.get(pattern, None), sheet_name='overall summary', header=0) except FileNotFoundError: - logging.error("File %s not found.", self.RESULT_EXCEL.get(pattern,None)) + logging.error("File %s not found.", self.RESULT_EXCEL.get(pattern, None)) return for index, row in df.iterrows(): @@ -180,7 +180,7 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): self.assertEqual(duration[index], row["Duration(ms)"]) self.assertEqual(duration_ratio[index], row["Duration Ratio"]) - soup = BeautifulSoup(open(self.RESULT_HTML.get(pattern,None)), 'html.parser') + soup = BeautifulSoup(open(self.RESULT_HTML.get(pattern, None)), 'html.parser') for h2 in soup.find_all('h2'): if h2.contents[0] == "overall summary": div_content = h2.next.next.next @@ -194,52 +194,58 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): def test_all_bandwidth_contention_analysis(self): bandwidth_contention_analysis = [ - "hcom_allGather__508_1_1","hcom_allGather__508_4_1","hcom_allGather__508_8_1", - "hcom_allGather__508_108_1","hcom_allGather__508_112_1","hcom_allGather__508_113_1", - "hcom_allGather__508_137_1","hcom_allGather__508_141_1","hcom_allGather__508_145_1", - "hcom_allGather__508_153_1","hcom_allGather__508_157_1","hcom_allGather__508_173_1", - "hcom_allGather__508_177_1","hcom_allGather__508_181_1","hcom_allGather__508_209_1", - "hcom_reduceScatter__868_261_1","hcom_reduceScatter__868_266_1","hcom_allGather__508_276_1", - "hcom_reduceScatter__508_283_1","hcom_reduceScatter__508_291_1","hcom_reduceScatter__508_299_1", - "hcom_reduceScatter__508_307_1","hcom_allGather__508_308_1","hcom_reduceScatter__508_315_1", - "hcom_reduceScatter__508_323_1","hcom_reduceScatter__508_331_1","hcom_reduceScatter__508_339_1", - "hcom_reduceScatter__508_347_1","hcom_reduceScatter__508_355_1","hcom_allGather__508_356_1", - "hcom_reduceScatter__508_363_1","hcom_reduceScatter__508_371_1","hcom_allGather__508_372_1", - "hcom_reduceScatter__508_379_1","hcom_reduceScatter__508_387_1","hcom_allGather__508_388_1", - "hcom_reduceScatter__508_395_1","hcom_reduceScatter__508_403_1","hcom_allGather__508_404_1", - "hcom_reduceScatter__508_411_1","hcom_reduceScatter__508_419_1","hcom_reduceScatter__508_427_1", - "hcom_reduceScatter__508_435_1","hcom_reduceScatter__508_443_1","hcom_reduceScatter__508_451_1", - "hcom_reduceScatter__508_459_1","hcom_reduceScatter__508_467_1","hcom_allGather__508_468_1", - "hcom_reduceScatter__508_475_1","hcom_reduceScatter__508_483_1","hcom_reduceScatter__508_491_1", - "hcom_reduceScatter__508_499_1","hcom_reduceScatter__508_507_1","hcom_reduceScatter__508_515_1", - "hcom_allGather__508_516_1","hcom_reduceScatter__508_523_1","hcom_reduceScatter__508_531_1", - "hcom_reduceScatter__508_539_1","hcom_reduceScatter__508_547_1","hcom_reduceScatter__508_555_1", - "hcom_reduceScatter__508_563_1","hcom_reduceScatter__508_571_1","hcom_reduceScatter__508_579_1", - "hcom_reduceScatter__508_587_1","hcom_allGather__508_588_1","hcom_reduceScatter__508_595_1", - "hcom_reduceScatter__508_603_1","hcom_reduceScatter__508_611_1","hcom_reduceScatter__508_619_1", - "hcom_reduceScatter__508_627_1","hcom_reduceScatter__508_635_1","hcom_reduceScatter__508_643_1", - "hcom_allGather__508_644_1","hcom_reduceScatter__508_651_1","hcom_reduceScatter__508_659_1", - "hcom_reduceScatter__508_667_1","hcom_reduceScatter__508_675_1","hcom_reduceScatter__508_683_1" + "hcom_allGather__508_1_1", "hcom_allGather__508_4_1", "hcom_allGather__508_8_1", + "hcom_allGather__508_108_1", "hcom_allGather__508_112_1", "hcom_allGather__508_113_1", + "hcom_allGather__508_137_1", "hcom_allGather__508_141_1", "hcom_allGather__508_145_1", + "hcom_allGather__508_153_1", "hcom_allGather__508_157_1", "hcom_allGather__508_173_1", + "hcom_allGather__508_177_1", "hcom_allGather__508_181_1", "hcom_allGather__508_209_1", + "hcom_reduceScatter__868_261_1", "hcom_reduceScatter__868_266_1", "hcom_allGather__508_276_1", + "hcom_reduceScatter__508_283_1", "hcom_reduceScatter__508_291_1", "hcom_reduceScatter__508_299_1", + "hcom_reduceScatter__508_307_1", "hcom_allGather__508_308_1", "hcom_reduceScatter__508_315_1", + "hcom_reduceScatter__508_323_1", "hcom_reduceScatter__508_331_1", "hcom_reduceScatter__508_339_1", + "hcom_reduceScatter__508_347_1", "hcom_reduceScatter__508_355_1", "hcom_allGather__508_356_1", + "hcom_reduceScatter__508_363_1", "hcom_reduceScatter__508_371_1", "hcom_allGather__508_372_1", + "hcom_reduceScatter__508_379_1", "hcom_reduceScatter__508_387_1", "hcom_allGather__508_388_1", + "hcom_reduceScatter__508_395_1", "hcom_reduceScatter__508_403_1", "hcom_allGather__508_404_1", + "hcom_reduceScatter__508_411_1", "hcom_reduceScatter__508_419_1", "hcom_reduceScatter__508_427_1", + "hcom_reduceScatter__508_435_1", "hcom_reduceScatter__508_443_1", "hcom_reduceScatter__508_451_1", + "hcom_reduceScatter__508_459_1", "hcom_reduceScatter__508_467_1", "hcom_allGather__508_468_1", + "hcom_reduceScatter__508_475_1", "hcom_reduceScatter__508_483_1", "hcom_reduceScatter__508_491_1", + "hcom_reduceScatter__508_499_1", "hcom_reduceScatter__508_507_1", "hcom_reduceScatter__508_515_1", + "hcom_allGather__508_516_1", "hcom_reduceScatter__508_523_1", "hcom_reduceScatter__508_531_1", + "hcom_reduceScatter__508_539_1", "hcom_reduceScatter__508_547_1", "hcom_reduceScatter__508_555_1", + "hcom_reduceScatter__508_563_1", "hcom_reduceScatter__508_571_1", "hcom_reduceScatter__508_579_1", + "hcom_reduceScatter__508_587_1", "hcom_allGather__508_588_1", "hcom_reduceScatter__508_595_1", + "hcom_reduceScatter__508_603_1", "hcom_reduceScatter__508_611_1", "hcom_reduceScatter__508_619_1", + "hcom_reduceScatter__508_627_1", "hcom_reduceScatter__508_635_1", "hcom_reduceScatter__508_643_1", + "hcom_allGather__508_644_1", "hcom_reduceScatter__508_651_1", "hcom_reduceScatter__508_659_1", + "hcom_reduceScatter__508_667_1", "hcom_reduceScatter__508_675_1", "hcom_reduceScatter__508_683_1" ] duration = [ - 8.3454,13.8113,39.8263,21.6036,38.2598,5.3913,13.4007,9.6871,8.8002,10.0535,8.3423,9.3205,11.3891, - 9.473,12.7247,19.4176,13.2621,16.3541,127.5414,127.288,126.6839,129.0707,11.8205,128.8378,130.0548, - 128.3927,124.9711,128.0221,122.8157,11.7839,127.0278,123.3328,11.9078,122.3141,123.1837,11.2561, - 123.8337,127.5955,11.5881,123.0412,128.4852,122.3674,127.1958,127.5779,129.6155,127.2981,125.5495, - 11.0916,127.4827,126.4632,125.0414,123.9187,125.168,127.1,12.6763,126.3728,126.9693,127.677, - 127.1439,127.2013,127.9102,125.7989,126.4961,127.6573,12.2088,127.6283,126.3803,129.8238,126.2997, - 127.4806,129.2007,127.2733,12.0963,126.8322,127.5317,126.482,127.8283,129.2951 + 8.3454, 13.8113, 39.8263, 21.6036, 38.2598, 5.3913, 13.4007, 9.6871, 8.8002, 10.0535, 8.3423, 9.3205, + 11.3891, + 9.473, 12.7247, 19.4176, 13.2621, 16.3541, 127.5414, 127.288, 126.6839, 129.0707, 11.8205, 128.8378, + 130.0548, + 128.3927, 124.9711, 128.0221, 122.8157, 11.7839, 127.0278, 123.3328, 11.9078, 122.3141, 123.1837, 11.2561, + 123.8337, 127.5955, 11.5881, 123.0412, 128.4852, 122.3674, 127.1958, 127.5779, 129.6155, 127.2981, 125.5495, + 11.0916, 127.4827, 126.4632, 125.0414, 123.9187, 125.168, 127.1, 12.6763, 126.3728, 126.9693, 127.677, + 127.1439, 127.2013, 127.9102, 125.7989, 126.4961, 127.6573, 12.2088, 127.6283, 126.3803, 129.8238, 126.2997, + 127.4806, 129.2007, 127.2733, 12.0963, 126.8322, 127.5317, 126.482, 127.8283, 129.2951 ] bandwidth = [ - 5.49,4.8,5.99,14.13,3.24,6.25,8.52,5.17,5.34,8.24,5.43,6.15,9.79,5.55,4.39,13.35,13.14,3.61,2.51, - 2.88,2.83,3.07,4.81,2.55,2.57,2.73,2.84,2.44,3.01,4.95,2.63,3.06,3.77,2.88,3.44,4.72,2.91,3.21, - 4.47,2.38,2.31,2.9,4.26,3.57,2.31,2.24,2.81,4.37,2.67,2.8,2.74,2.16,2.79,2.88,5.79,2.75,2.93,2.88, - 2.31,2.72,2.39,2.6,2.55,2.58,4.29,2.69,2.86,2.09,3.12,2.31,2.28,2.87,6.97,3.1,2.35,3.4,2.61,2.62 + 5.49, 4.8, 5.99, 14.13, 3.24, 6.25, 8.52, 5.17, 5.34, 8.24, 5.43, 6.15, 9.79, 5.55, 4.39, 13.35, 13.14, + 3.61, 2.51, + 2.88, 2.83, 3.07, 4.81, 2.55, 2.57, 2.73, 2.84, 2.44, 3.01, 4.95, 2.63, 3.06, 3.77, 2.88, 3.44, 4.72, 2.91, + 3.21, + 4.47, 2.38, 2.31, 2.9, 4.26, 3.57, 2.31, 2.24, 2.81, 4.37, 2.67, 2.8, 2.74, 2.16, 2.79, 2.88, 5.79, 2.75, + 2.93, 2.88, + 2.31, 2.72, 2.39, 2.6, 2.55, 2.58, 4.29, 2.69, 2.86, 2.09, 3.12, 2.31, 2.28, 2.87, 6.97, 3.1, 2.35, 3.4, + 2.61, 2.62 ] try: - df = pd.read_excel(self.RESULT_EXCEL.get("all",None), sheet_name='Bandwidth Contention Analysis', header=0) + df = pd.read_excel(self.RESULT_EXCEL.get("all", None), sheet_name='Bandwidth Contention Analysis', header=0) except FileNotFoundError: - logging.error("File %s not found.", self.RESULT_EXCEL.get("all",None)) + logging.error("File %s not found.", self.RESULT_EXCEL.get("all", None)) return for index, row in df.iterrows(): @@ -276,9 +282,9 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): test_pattern = ["all", "computation"] for pattern in test_pattern: try: - df = pd.read_excel(self.RESULT_EXCEL.get(pattern,None), sheet_name='AICPU operator', header=0) + df = pd.read_excel(self.RESULT_EXCEL.get(pattern, None), sheet_name='AICPU operator', header=0) except FileNotFoundError: - logging.error("File %s not found.", self.RESULT_EXCEL.get(pattern,None)) + logging.error("File %s not found.", self.RESULT_EXCEL.get(pattern, None)) return for index, row in df.iterrows(): @@ -293,7 +299,7 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): self.assertEqual(output_formats[index], row["output_formats"]) self.assertEqual(stack_info[index], math.isnan(row["stack_info"])) - soup = BeautifulSoup(open(self.RESULT_HTML.get(pattern,None)), 'html.parser') + soup = BeautifulSoup(open(self.RESULT_HTML.get(pattern, None)), 'html.parser') for h2 in soup.find_all('h2'): if h2.contents[0] == "AICPU Issues": div_content = h2.next.next.next @@ -324,17 +330,17 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): self.assertEqual(b_names[b_index], b_content.text) def test_Affinity_API(self): - affinity_api = ["torch_npu.npu_confusion_transpose","torch_npu.optim.NpuFusedAdamW"] - code_stacks = [True,True] - stack_called_counts = [True,True] + affinity_api = ["torch_npu.npu_confusion_transpose", "torch_npu.optim.NpuFusedAdamW"] + code_stacks = [True, True] + stack_called_counts = [True, True] ignore_api = ["torch_npu.optim.NpuFusedAdamW", "torch_npu.npu_confusion_transpose"] test_pattern = ["all", "schedule"] for pattern in test_pattern: try: - df = pd.read_excel(self.RESULT_EXCEL.get(pattern,None), sheet_name='Affinity apis', header=0) + df = pd.read_excel(self.RESULT_EXCEL.get(pattern, None), sheet_name='Affinity apis', header=0) except FileNotFoundError: - logging.error("File %s not found.", self.RESULT_EXCEL.get(pattern,None)) + logging.error("File %s not found.", self.RESULT_EXCEL.get(pattern, None)) return for index, row in df.iterrows(): @@ -342,12 +348,12 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): self.assertEqual(code_stacks[index], math.isnan(row["Code stacks"])) self.assertEqual(stack_called_counts[index], math.isnan(row["Stack called counts"])) - soup = BeautifulSoup(open(self.RESULT_HTML.get(pattern,None)), 'html.parser') + soup = BeautifulSoup(open(self.RESULT_HTML.get(pattern, None)), 'html.parser') for h2 in soup.find_all('h2'): if h2.contents[0] == "Affinity API Issues": div_content = h2.next.next.next - self.assertEqual(ignore_api[0],div_content.contents[-2].contents[-2].text) - self.assertEqual(ignore_api[1],div_content.contents[-2].contents[-4].text) + self.assertEqual(ignore_api[0], div_content.contents[-2].contents[-2].text) + self.assertEqual(ignore_api[1], div_content.contents[-2].contents[-4].text) def test_operator_dispatch(self): issues = ["operator dispatch"] @@ -364,9 +370,9 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): test_pattern = ["all", "schedule"] for pattern in test_pattern: try: - df = pd.read_excel(self.RESULT_EXCEL.get(pattern,None), sheet_name='operator dispatch', header=0) + df = pd.read_excel(self.RESULT_EXCEL.get(pattern, None), sheet_name='operator dispatch', header=0) except FileNotFoundError: - logging.error("File %s not found.", self.RESULT_EXCEL.get(pattern,None)) + logging.error("File %s not found.", self.RESULT_EXCEL.get(pattern, None)) return for index, row in df.iterrows(): self.assertEqual(issues[index], row["Issues"]) @@ -374,7 +380,7 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): self.assertEqual(counts[index], row["counts"]) self.assertEqual(total_time[index], round(row["total time"], 4)) - soup = BeautifulSoup(open(self.RESULT_HTML.get(pattern,None)), 'html.parser') + soup = BeautifulSoup(open(self.RESULT_HTML.get(pattern, None)), 'html.parser') for h2 in soup.find_all('h2'): if h2.contents[0] == "Operator Dispatch Issues": div_content = h2.next.next.next @@ -391,5 +397,3 @@ class TestAdvisorCmdSingleAscendPtNoCompare(TestCase): self.assertEqual(t1_issue[row_index - 1], row.find_all('td')[0].text) self.assertEqual(t1_counts[row_index - 1], row.find_all('td')[1].text) self.assertEqual(t1_elapsed_time[row_index - 1], row.find_all('td')[2].text) - - diff --git a/profiler/test/st/advisor/utils.py b/profiler/test/st/advisor/utils.py index 66718f388224eedaa1d8deff4bc7ddf01b32115d..2293907960bd36504a1485a6e91a0b3525be7dff 100644 --- a/profiler/test/st/advisor/utils.py +++ b/profiler/test/st/advisor/utils.py @@ -6,11 +6,13 @@ import subprocess RE_EXCEL_MATCH_EXP = r"^mstt_advisor_\d{1,20}\.xlsx" RE_HTML_MATCH_EXP = r"^mstt_advisor_\d{1,20}\.html" + def execute_cmd(cmd): - logging.info('Execute command:%s'," ".join(cmd)) + logging.info('Execute command:%s', " ".join(cmd)) completed_process = subprocess.run(cmd, capture_output=True, shell=False, check=True) return completed_process.returncode + def get_files(out_path): dirs = os.listdir(out_path) result_html = {} @@ -44,6 +46,3 @@ def get_files(out_path): result_html[pattern] = os.path.join(files_out_path, newest_html_file) result_excel[pattern] = os.path.join(log_dir, newest_excel_file) return result_html, result_excel - - - diff --git a/profiler/test/ut/advisor/common/test_enum_params_parser.py b/profiler/test/ut/advisor/common/test_enum_params_parser.py index 8e5ddb680444c944898201bb58f7b71a520b924b..c27a632f64c844eaa4fa079214d87a2caae022ca 100644 --- a/profiler/test/ut/advisor/common/test_enum_params_parser.py +++ b/profiler/test/ut/advisor/common/test_enum_params_parser.py @@ -17,7 +17,7 @@ class TestEnumParamsParser(unittest.TestCase): def setUp(self) -> None: self.enum_params_parser = EnumParamsParser() - self.argument_keys = sorted(["cann_version", "torch_version", "analysis_dimensions", "profiling_type"]) + self.argument_keys = sorted(["cann_version", "torch_version", "analysis_dimensions", "profiling_type", "mindspore_version"]) self.env_keys = ["ADVISOR_ANALYZE_PROCESSES", "DISABLE_PROFILING_COMPARISON", "DISABLE_AFFINITY_API"] def test_get_keys(self):