From d98c98e7def878cf2ed514cccfa6744302e6b521 Mon Sep 17 00:00:00 2001 From: huxianglong Date: Mon, 23 Dec 2024 11:08:18 +0800 Subject: [PATCH 1/2] change compare log --- .../dispatch/timeline_op_dispatch_analyzer.py | 2 +- .../profiling_parser/npu_profiling_parser.py | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py b/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py index 1a5af994b..2e6e0e93b 100644 --- a/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py +++ b/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py @@ -53,7 +53,7 @@ class OpDispatchAnalyzer(BaseAnalyzer): :return: result """ if "mindspore" in self.profiling_type: - logger.warning("The analyzer %s does not support MindSpore.", self.__class__.__name__) + logger.info("The analyzer %s does not support MindSpore.", self.__class__.__name__) return self.result self.get_op_compile_info(self.dataset) self.make_record(self.result) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index cdb168630..04cf22e54 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -112,6 +112,8 @@ class NPUProfilingParser(BaseProfilingParser): return list(func_list) def _update_kernel_details(self): + if self._path_level == Constant.TRACE_PATH: + return if self._args.use_kernel_type: op_statistics = self._read_csv_data(self._op_statistic_path, OpStatisticBean) if not op_statistics: @@ -143,6 +145,8 @@ class NPUProfilingParser(BaseProfilingParser): self._result_data.update_kernel_details(kernels_dict) def _update_memory_list(self): + if self._path_level == Constant.TRACE_PATH: + return memory_data = self._read_csv_data(self._operator_memory_path, OperatorMemoryBean) if memory_data: self._dequeue_data.sort(key=lambda x: x.start_time) @@ -165,7 +169,9 @@ class NPUProfilingParser(BaseProfilingParser): Constant.RELEASE_TIME: data.release_time}) def _update_kernel_dict(self): - kernel_details = self._read_csv_data(self._kernel_detail_path, KernelDetailsBean) + kernel_details=[] + if self._path_level != Constant.TRACE_PATH: + kernel_details = self._read_csv_data(self._kernel_detail_path, KernelDetailsBean) input_shape_dict = {kernel.start_time: kernel.input_shapes for kernel in kernel_details} for kernel in self._all_kernels.values(): input_shape = input_shape_dict.get(kernel.start_time, "") @@ -189,7 +195,8 @@ class NPUProfilingParser(BaseProfilingParser): try: communication_json = FileManager.read_json_file(self._communication_path) except FileNotFoundError: - logger.warning("The file communication.json does not exist.") + if self._path_level != Constant.TRACE_PATH: + logger.warning("The file communication.json does not exist.") return except Exception: logger.error("Failed to read communication.json.") @@ -390,8 +397,10 @@ class NPUProfilingParser(BaseProfilingParser): self._result_data.overall_metrics.update_lccl_info(event.dur) def __parse_kernel_csv(self): + kernel_details=[] try: - kernel_details = self._read_csv_data(self._kernel_detail_path, KernelDetailsBean) + if self._path_level != Constant.TRACE_PATH: + kernel_details = self._read_csv_data(self._kernel_detail_path, KernelDetailsBean) except Exception: logger.error('Npu kernel details csv file is not available.') return @@ -408,8 +417,10 @@ class NPUProfilingParser(BaseProfilingParser): self.categorize_computing_performance_data(event, flow_start_time) def __parse_mem_csv(self): + memory_record=[] try: - memory_record = self._read_csv_data(self._memory_record_path, MemoryRecordBean) + if self._path_level != Constant.TRACE_PATH: + memory_record = self._read_csv_data(self._memory_record_path, MemoryRecordBean) except FileNotFoundError: logger.warning('Npu memory record csv file is not available.') return -- Gitee From eb11887f30e596ac2846b4658bcb043c57a95a98 Mon Sep 17 00:00:00 2001 From: huxianglong Date: Mon, 23 Dec 2024 17:16:28 +0800 Subject: [PATCH 2/2] log --- .../profiling_parser/base_profiling_parser.py | 11 +++++++++++ .../profiling_parser/npu_profiling_parser.py | 8 -------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py index 98b5cd0bc..acf07b740 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py @@ -64,6 +64,7 @@ class BaseProfilingParser(ABC): def __init__(self, args: any, path_dict: dict, step_id: int = Constant.VOID_STEP): self._args = args + self._path_level = self._get_path_level(path_dict) self._profiling_type = path_dict.get(Constant.PROFILING_TYPE) self._profiling_path = path_dict.get(Constant.PROFILING_PATH) self._json_path = path_dict.get(Constant.TRACE_PATH) @@ -96,6 +97,14 @@ class BaseProfilingParser(ABC): self._cpu_cube_op = cpu_cube_op return self._cpu_cube_op + @staticmethod + def _get_path_level(path_dict): + if path_dict.get(Constant.PROFILING_PATH, "") == path_dict.get(Constant.TRACE_PATH, ""): + return Constant.TRACE_PATH + if path_dict.get(Constant.PROFILING_PATH, "") == path_dict.get(Constant.ASCEND_OUTPUT_PATH, ""): + return Constant.ASCEND_OUTPUT_PATH + return Constant.PROFILING_PATH + @abstractmethod def _update_kernel_details(self): raise NotImplementedError("Function _update_kernel_details need to be implemented.") @@ -325,6 +334,8 @@ class BaseProfilingParser(ABC): task_index += 1 def _check_result_data(self): + if self._path_level == Constant.TRACE_PATH: + return if self._enable_operator_compare or self._enable_memory_compare or self._enable_api_compare: if not self._result_data.torch_op_data: logger.warning("Can't find any torch op in the file: %s", self._profiling_path) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 04cf22e54..e78bc83f6 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -24,7 +24,6 @@ class NPUProfilingParser(BaseProfilingParser): def __init__(self, args: any, path_dict: dict, step_id: int = Constant.VOID_STEP): super().__init__(args, path_dict, step_id) - self._path_level = self._get_path_level(path_dict) self._operator_memory_path = os.path.join(path_dict.get(Constant.ASCEND_OUTPUT_PATH, ""), "operator_memory.csv") self._memory_record_path = os.path.join(path_dict.get(Constant.ASCEND_OUTPUT_PATH, ""), "memory_record.csv") self._kernel_detail_path = os.path.join(path_dict.get(Constant.ASCEND_OUTPUT_PATH, ""), "kernel_details.csv") @@ -47,13 +46,6 @@ class NPUProfilingParser(BaseProfilingParser): self._enable_api_compare, self._enable_communication_compare)): self._filter_meta_id() - @staticmethod - def _get_path_level(path_dict): - if path_dict.get(Constant.PROFILING_PATH, "") == path_dict.get(Constant.TRACE_PATH, ""): - return Constant.TRACE_PATH - if path_dict.get(Constant.PROFILING_PATH, "") == path_dict.get(Constant.ASCEND_OUTPUT_PATH, ""): - return Constant.ASCEND_OUTPUT_PATH - return Constant.PROFILING_PATH @staticmethod def __calculate_overlap_time_with_uncovered_communication(uncovered_communication_events: list, events: list): -- Gitee