From 70df980acb20bc4524bfb15dca527f5a11eb201e Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 18 Jul 2025 10:37:34 +0800 Subject: [PATCH 1/7] compare first_diff_analyze --- .../msprobe/core/common/utils.py | 6 + .../msprobe/core/compare/acc_compare.py | 26 +++-- .../msprobe/core/compare/config.py | 5 +- .../core/compare/diff_analyze/__init__.py | 0 .../diff_analyze/diff_analyze_threshold.yaml | 14 +++ .../diff_analyze/first_diff_analyze.py | 108 ++++++++++++++++++ .../msprobe/core/compare/highlight.py | 60 +--------- .../msprobe/core/compare/utils.py | 55 +++++++++ .../msprobe/pytorch/compare/pt_compare.py | 2 +- .../pytorch/compare/pt_diff_analyze.py | 21 ++++ .../core_ut/compare/test_acc_compare_utils.py | 84 +++++++++++++- .../compare/test_cmp_first_diff_analyze.py | 106 +++++++++++++++++ .../core_ut/compare/test_cmp_highlight.py | 91 +-------------- 13 files changed, 420 insertions(+), 158 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/core/compare/diff_analyze/__init__.py create mode 100644 debug/accuracy_tools/msprobe/core/compare/diff_analyze/diff_analyze_threshold.yaml create mode 100644 debug/accuracy_tools/msprobe/core/compare/diff_analyze/first_diff_analyze.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/compare/pt_diff_analyze.py create mode 100644 debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_first_diff_analyze.py diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 5645cba69..04acfb01f 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -82,6 +82,8 @@ class MsprobeBaseException(Exception): INVALID_STATE_ERROR = 35 INVALID_API_NAME_ERROR = 36 CROSS_FRAME_ERROR = 37 + MISSING_THRESHOLD_ERROR = 38 + WRONG_THRESHOLD_ERROR = 38 def __init__(self, code, error_info: str = ""): super(MsprobeBaseException, self).__init__() @@ -283,6 +285,10 @@ def add_time_with_xlsx(name): return '{}_{}.xlsx'.format(name, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) +def add_time_with_json(name): + return '{}_{}.json'.format(name, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) + + def add_time_with_yaml(name): return '{}_{}.yaml'.format(name, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 7b34b73d2..e439856c6 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -25,16 +25,18 @@ from tqdm import tqdm from msprobe.core.advisor.advisor import Advisor from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_utils import load_json, remove_path, create_directory +from msprobe.core.common.file_utils import load_json, remove_path, create_directory, save_json from msprobe.core.common.log import logger from msprobe.core.common.utils import CompareException, add_time_with_xlsx, check_op_str_pattern_valid, \ - set_dump_path, get_dump_mode, check_compare_param, check_configuration_param, load_stack_json, get_file_type + set_dump_path, get_dump_mode, check_compare_param, check_configuration_param, load_stack_json, get_file_type, \ + add_time_with_json from msprobe.core.compare.check import check_dump_json_str, check_stack_json_str, cross_dtype_mapping from msprobe.core.compare.utils import merge_tensor, print_compare_ends_info, read_op, \ reorder_op_x_list, set_stack_json_path, check_api_info_len from msprobe.core.compare.config import ModeConfig, MappingConfig, MappingDict from msprobe.core.compare.multiprocessing_compute import CompareRealData from msprobe.core.compare.highlight import HighLight +from msprobe.core.compare.diff_analyze.first_diff_analyze import FirstDiffAnalyze @dataclass @@ -49,6 +51,7 @@ class ComparisonConfig: api_mapping: dict layer_mapping: dict compared_file_type: str + first_diff_analyze: bool class Comparator: @@ -57,17 +60,18 @@ class Comparator: self.mode_config = mode_config self.mapping_config = mapping_config self.cross_frame = is_cross_framework - self.mapping_dict = MappingDict(mapping_config) - @staticmethod - def process_output_file(output_path, suffix, compared_file_type): + def process_output_file(self, output_path, suffix, compared_file_type): file_name_prefix_mapping = { Const.DUMP_JSON_FILE: "compare_result", Const.DEBUG_JSON_FILE: "debug_compare_result" } file_name_prefix = file_name_prefix_mapping.get(compared_file_type, "compare_result") - file_name = add_time_with_xlsx(file_name_prefix + suffix) + if self.mode_config.first_diff_analyze: + file_name = add_time_with_json("compare_result" + suffix) + else: + file_name = add_time_with_xlsx("compare_result" + suffix) file_path = os.path.join(os.path.realpath(output_path), file_name) if os.path.exists(file_path): logger.warning(f"{file_path} will be deleted.") @@ -108,6 +112,13 @@ class Comparator: logger.warning("Can`t match any op. No compare result file generated.") return + if self.mode_config.first_diff_analyze: + first_diff_analyze = FirstDiffAnalyze() + check_result = first_diff_analyze.check(result_df) + save_json(file_path, check_result) + logger.info(f"Saving json file to disk: {file_path}") + return + # compare real data if self.mode_config.dump_mode == Const.ALL: compare_real_data = CompareRealData(self.file_reader, self.mode_config, self.cross_frame) @@ -701,7 +712,7 @@ class CalcStatsDiff: condition_md5_equal = result_df[CompareConst.NPU_MD5] == result_df[CompareConst.BENCH_MD5] result_df.loc[condition_md5_equal, CompareConst.RESULT] = CompareConst.PASS result_df.loc[~condition_md5_equal & ~condition_no_bench, CompareConst.RESULT] = CompareConst.DIFF - elif self.mode_config.dump_mode == Const.SUMMARY: + elif self.mode_config.first_diff_analyze or self.mode_config.dump_mode == Const.SUMMARY: warning_list = [ self.calc_summary_diff(result_df, condition_no_bench, stats_index) for stats_index in ['max', 'min', 'mean', 'l2norm'] @@ -734,6 +745,7 @@ def setup_comparison(input_param, output_path, **kwargs) -> ComparisonConfig: cell_mapping=kwargs.get('cell_mapping', {}), api_mapping=kwargs.get('api_mapping', {}), layer_mapping=kwargs.get('layer_mapping', {}), + first_diff_analyze=kwargs.get('first_diff_analyze', False), compared_file_type='', ) diff --git a/debug/accuracy_tools/msprobe/core/compare/config.py b/debug/accuracy_tools/msprobe/core/compare/config.py index 448139b8b..72942400c 100644 --- a/debug/accuracy_tools/msprobe/core/compare/config.py +++ b/debug/accuracy_tools/msprobe/core/compare/config.py @@ -21,11 +21,12 @@ from msprobe.core.common.file_utils import load_yaml class ModeConfig: def __init__(self, stack_mode=False, auto_analyze=True, fuzzy_match=False, dump_mode=Const.SUMMARY, - compared_file_type=Const.DUMP_JSON_FILE): + first_diff_analyze=False, compared_file_type=Const.DUMP_JSON_FILE): self.stack_mode = stack_mode self.auto_analyze = auto_analyze self.fuzzy_match = fuzzy_match self.dump_mode = dump_mode + self.first_diff_analyze = first_diff_analyze self.compared_file_type = compared_file_type @@ -69,4 +70,4 @@ class MappingDict: else: raise TypeError(f"The type of parameter `data_mapping` must be dict, str or None, but got " f"{type(data_mapping)}") - return data_mapping_dict \ No newline at end of file + return data_mapping_dict diff --git a/debug/accuracy_tools/msprobe/core/compare/diff_analyze/__init__.py b/debug/accuracy_tools/msprobe/core/compare/diff_analyze/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/core/compare/diff_analyze/diff_analyze_threshold.yaml b/debug/accuracy_tools/msprobe/core/compare/diff_analyze/diff_analyze_threshold.yaml new file mode 100644 index 000000000..4b35d10bf --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/compare/diff_analyze/diff_analyze_threshold.yaml @@ -0,0 +1,14 @@ +compare_metrics: + - MaxRelativeErr + - MinRelativeErr + - MeanRelativeErr + - NormRelativeErr + +MaxRelativeErr: + - 0.5 +MinRelativeErr: + - 0.5 +MeanRelativeErr: + - 0.5 +NormRelativeErr: + - 0.5 \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/compare/diff_analyze/first_diff_analyze.py b/debug/accuracy_tools/msprobe/core/compare/diff_analyze/first_diff_analyze.py new file mode 100644 index 000000000..00ecacc08 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/compare/diff_analyze/first_diff_analyze.py @@ -0,0 +1,108 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from msprobe.core.common.utils import safe_get_value, logger, CompareException +from msprobe.core.common.file_utils import load_yaml +from msprobe.core.compare.utils import api_batches_update, get_name_and_state + +cur_dir = os.path.dirname(os.path.realpath(__file__)) +diff_threshold_yaml_path = os.path.join(cur_dir, 'diff_analyze_threshold.yaml') +thresholds = load_yaml(diff_threshold_yaml_path) +cmp_metrics = thresholds.get('compare_metrics') + + +class FirstDiffAnalyze: + @staticmethod + def single_metric_diff_check(cmp_metric, metric_value): + threshold = thresholds.get(cmp_metric, None) + if threshold is None: + logger.error(f"Check diff or {cmp_metric} need to configure the threshold. " + f"Please configure it in 'diff_analyze_threshold.yaml'.") + raise CompareException(CompareException.MISSING_THRESHOLD_ERROR) + if not isinstance(threshold, list) or len(threshold) != 1: + logger.error(f"{cmp_metric} threshold configure wrong. Please check.") + raise CompareException(CompareException.WRONG_THRESHOLD_ERROR) + if isinstance(metric_value, str) and metric_value.endswith('%'): + metric_value_float = float(metric_value[:-1]) / 100 + if metric_value_float > threshold[0]: + return True + return False + + def single_api_check(self, result_slice, header): + """ + 单个api差异检查 + + :param result_slice: 数据切片 + :param header: 列名列表 + :return: {'is_same': bool, 'op_items': list[dict]} + """ + single_check_result = { + 'is_same': True, + 'op_items': [] + } + + column_indices = {name: idx for idx, name in enumerate(header)} + + for line in result_slice: + op_item = { + column_name: line[column_indices[column_name]] + for column_name in header + } + single_check_result['op_items'].append(op_item) + + for cmp_metric in cmp_metrics: + metric_value = line[column_indices[cmp_metric]] + if self.single_metric_diff_check(cmp_metric, metric_value): + single_check_result['is_same'] = False + break + return single_check_result + + def check(self, result_df): + """ + 比对后循环遍历api检查norm差异 + example： + { + 'Functional.conv2d.0.forward': { + 'is_same': true, + 'op_items': [ + { + 'NPU name': 'Functional.conv2d.0.forward.input.0', + 'Bench name': 'Functional.conv2d.0.forward.input.0', + 'xxx': 1, + 'NormRelativeErr': 2, + 'yyy': 3, + ... + } + ] + } + } + """ + result = result_df.values + header = result_df.columns.tolist() + + api_batches = [] + for i, res_i in enumerate(result): + api_full_name = safe_get_value(res_i, 0, "res_i") + api_name, state = get_name_and_state(api_full_name) + api_batches_update(api_batches, api_name, state, i) + + check_result = {} + for api_batch in api_batches: + result_slice = result[api_batch.start: api_batch.params_grad_end_index] + check_result[api_batch.api_name[: -1]] = self.single_api_check(result_slice, header) + + return check_result \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/compare/highlight.py b/debug/accuracy_tools/msprobe/core/compare/highlight.py index 37eec4216..6bdcec560 100644 --- a/debug/accuracy_tools/msprobe/core/compare/highlight.py +++ b/debug/accuracy_tools/msprobe/core/compare/highlight.py @@ -16,7 +16,6 @@ import abc import math import multiprocessing -import re from collections import namedtuple import numpy as np @@ -29,7 +28,7 @@ from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.file_utils import save_workbook from msprobe.core.common.log import logger from msprobe.core.common.utils import get_header_index, safe_get_value -from msprobe.core.compare.utils import table_value_is_valid, CompareException +from msprobe.core.compare.utils import table_value_is_valid, api_batches_update from msprobe.core.compare.config import ModeConfig @@ -160,65 +159,10 @@ class HighlightRules: } -class ApiBatch: - def __init__(self, api_name: str, start: int): - self.api_name = api_name - self.start = start - self.input_len = 1 # input的数量 - self.params_end_index = start + 1 # params的结束index - self.output_end_index = start + 1 # output的结束index - self.params_grad_end_index = start + 1 # params_grad的结束index - # 内部state的标志("input", "output", "parameters", "parameters_grad"), - # 用于控制计算input_len, output_end_index, params_end_index, self.params_grad_end_index - self._state = Const.INPUT # api_batch初始化为input - - def set_state(self, state: str): - """设置当前状态""" - if state in {Const.INPUT, Const.OUTPUT, Const.KWARGS, Const.PARAMS, Const.PARAMS_GRAD}: - self._state = state - else: - raise ValueError(f"Invalid state: {state}") - - def increment(self, state: str): - self.set_state(state) - if self._state == Const.INPUT or self._state == Const.KWARGS: - self.input_len += 1 - self.params_end_index += 1 - self.output_end_index += 1 - if self._state == Const.PARAMS: - self.params_end_index += 1 - self.output_end_index += 1 - if self._state == Const.OUTPUT: - self.output_end_index += 1 - self.params_grad_end_index += 1 - - class HighLight: def __init__(self, mode_config: ModeConfig): self.mode_config = mode_config - @staticmethod - def api_batches_update(api_batches, api_name, state, index): - """ - 当一个api的所有item更新完后，input, output的索引范围： - input: [start: start+input_len] - output: [start+input_len: output_end_index] - params: [output_end_index: params_end_index] - """ - if not api_batches: - api_batches.append(ApiBatch(api_name, index)) - else: - api_batch = api_batches[-1] - if api_batch.api_name == api_name or ( - not re.search(Const.REGEX_FORWARD_BACKWARD, api_name) and api_name in api_batch.api_name): - try: - api_batch.increment(state) - except ValueError as e: - logger.error(f"api_batch: {api_batch} with invalid state, please check! {e}") - raise CompareException(CompareException.INVALID_STATE_ERROR) from e - else: - api_batches.append(ApiBatch(api_name, index)) - @staticmethod def check_indices_numeric(api_items, indices: list): """检查指定索引处的值是否都为数字类型（int 或 float）""" @@ -277,7 +221,7 @@ class HighLight: for i, res_i in enumerate(result): api_name = safe_get_value(res_i, -1, "res_i") # 内部定义倒数第一个元素必是api_origin_name state = safe_get_value(res_i, -2, "res_i") # 内部定义倒数第二个元素必是state - self.api_batches_update(api_batches, api_name, state, i) + api_batches_update(api_batches, api_name, state, i) with tqdm(total=len(api_batches), desc="API/Module Analyse Progress", unit="item", ncols=100) as progress_bar: for api_batch in api_batches: self.find_error_rows(result[api_batch.start: api_batch.params_grad_end_index], api_batch, diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 0f5ea4a50..4237b433d 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -277,6 +277,61 @@ def table_value_is_valid(value: str) -> bool: return True +class ApiBatch: + def __init__(self, api_name: str, start: int): + self.api_name = api_name + self.start = start + self.input_len = 1 # input的数量 + self.params_end_index = start + 1 # params的结束index + self.output_end_index = start + 1 # output的结束index + self.params_grad_end_index = start + 1 # params_grad的结束index + # 内部state的标志("input", "output", "parameters", "parameters_grad"), + # 用于控制计算input_len, output_end_index, params_end_index, self.params_grad_end_index + self._state = Const.INPUT # api_batch初始化为input + + def set_state(self, state: str): + """设置当前状态""" + if state in {Const.INPUT, Const.OUTPUT, Const.KWARGS, Const.PARAMS, Const.PARAMS_GRAD}: + self._state = state + else: + raise ValueError(f"Invalid state: {state}") + + def increment(self, state: str): + self.set_state(state) + if self._state == Const.INPUT or self._state == Const.KWARGS: + self.input_len += 1 + self.params_end_index += 1 + self.output_end_index += 1 + if self._state == Const.PARAMS: + self.params_end_index += 1 + self.output_end_index += 1 + if self._state == Const.OUTPUT: + self.output_end_index += 1 + self.params_grad_end_index += 1 + + +def api_batches_update(api_batches, api_name, state, index): + """ + 当一个api的所有item更新完后，input, output的索引范围： + input: [start: start+input_len] + output: [start+input_len: output_end_index] + params: [output_end_index: params_end_index] + """ + if not api_batches: + api_batches.append(ApiBatch(api_name, index)) + else: + api_batch = api_batches[-1] + if api_batch.api_name == api_name or ( + not re.search(Const.REGEX_FORWARD_BACKWARD, api_name) and api_name in api_batch.api_name): + try: + api_batch.increment(state) + except ValueError as e: + logger.error(f"api_batch: {api_batch} with invalid state, please check! {e}") + raise CompareException(CompareException.INVALID_STATE_ERROR) from e + else: + api_batches.append(ApiBatch(api_name, index)) + + def reorder_op_name_list(op_name_list, state_list): if not op_name_list: return op_name_list, state_list diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index b73c52dce..8d73f6f79 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -32,7 +32,7 @@ def compare(input_param, output_path, **kwargs): config = setup_comparison(input_param, output_path, **kwargs) mode_config = ModeConfig(config.stack_mode, config.auto_analyze, config.fuzzy_match, - config.dump_mode, config.compared_file_type) + config.dump_mode, config.first_diff_analyze, config.compared_file_type) mapping_config = MappingConfig(data_mapping=config.data_mapping) pt_comparator = Comparator(read_real_data, mode_config, mapping_config) pt_comparator.compare_core(input_param, output_path, suffix=config.suffix) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_diff_analyze.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_diff_analyze.py new file mode 100644 index 000000000..b558a20b6 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_diff_analyze.py @@ -0,0 +1,21 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from msprobe.pytorch.compare.distributed_compare import compare_distributed + + +def pt_diff_analyze(npu_dump_dir, bench_dump_dir, output_path, first_diff_analyze): + compare_distributed(npu_dump_dir, bench_dump_dir, output_path, first_diff_analyze=first_diff_analyze) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py index 4d264dd74..fcca0af96 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py @@ -13,7 +13,7 @@ from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.utils import CompareException from msprobe.core.compare.utils import ApiItemInfo, _compare_parser, check_and_return_dir_contents, extract_json, \ count_struct, get_accuracy, get_rela_diff_summary_mode, merge_tensor, op_item_parse, read_op, result_item_init, \ - stack_column_process, table_value_is_valid, reorder_op_name_list, reorder_op_x_list, gen_op_item + stack_column_process, table_value_is_valid, reorder_op_name_list, reorder_op_x_list, gen_op_item, ApiBatch # test_read_op_1 op_data = { @@ -786,3 +786,85 @@ class TestGenOpItem(unittest.TestCase): expected_md5 = f"{zlib.crc32(str(op_data['value']).encode()):08x}" self.assertEqual(result['md5'], expected_md5) self.assertEqual(result['state'], 'input') + + +class TestApiBatch(unittest.TestCase): + def test_ApiBatch_increment_input(self): + api_name = "functional.conv2d" + start = 2 + api_batch = ApiBatch(api_name, start) + + api_batch.increment(Const.INPUT) + + self.assertEqual(api_batch._state, Const.INPUT) + self.assertEqual(api_batch.input_len, 2) + self.assertEqual(api_batch.params_end_index, 4) + self.assertEqual(api_batch.output_end_index, 4) + self.assertEqual(api_batch.params_grad_end_index, 4) + + def test_ApiBatch_increment_output(self): + api_name = "functional.conv2d" + start = 2 + api_batch = ApiBatch(api_name, start) + + api_batch.increment(Const.OUTPUT) + + self.assertEqual(api_batch._state, Const.OUTPUT) + self.assertEqual(api_batch.input_len, 1) + self.assertEqual(api_batch.params_end_index, 3) + self.assertEqual(api_batch.output_end_index, 4) + self.assertEqual(api_batch.params_grad_end_index, 4) + + def test_ApiBatch_increment_kwargs(self): + api_name = "functional.conv2d" + start = 2 + api_batch = ApiBatch(api_name, start) + + api_batch.increment(Const.KWARGS) + + self.assertEqual(api_batch._state, Const.KWARGS) + self.assertEqual(api_batch.input_len, 2) + self.assertEqual(api_batch.params_end_index, 4) + self.assertEqual(api_batch.output_end_index, 4) + self.assertEqual(api_batch.params_grad_end_index, 4) + + def test_ApiBatch_increment_params(self): + api_name = "functional.conv2d" + start = 2 + api_batch = ApiBatch(api_name, start) + + api_batch.increment(Const.PARAMS) + + self.assertEqual(api_batch._state, Const.PARAMS) + self.assertEqual(api_batch.input_len, 1) + self.assertEqual(api_batch.params_end_index, 4) + self.assertEqual(api_batch.output_end_index, 4) + self.assertEqual(api_batch.params_grad_end_index, 4) + + def test_ApiBatch_increment_multiple_input(self): + api_name = "functional.conv2d" + start = 2 + api_batch = ApiBatch(api_name, start) + + api_batch.increment(Const.INPUT) + api_batch.increment(Const.INPUT) + + self.assertEqual(api_batch._state, Const.INPUT) + self.assertEqual(api_batch.input_len, 3) + self.assertEqual(api_batch.params_end_index, 5) + self.assertEqual(api_batch.output_end_index, 5) + self.assertEqual(api_batch.params_grad_end_index, 5) + + def test_ApiBatch_increment_multiple_output(self): + api_name = "functional.conv2d" + start = 2 + api_batch = ApiBatch(api_name, start) + + api_batch.increment(Const.OUTPUT) + api_batch.increment(Const.OUTPUT) + + self.assertEqual(api_batch._state, Const.OUTPUT) + self.assertEqual(api_batch.input_len, 1) + self.assertEqual(api_batch.params_end_index, 3) + self.assertEqual(api_batch.output_end_index, 5) + self.assertEqual(api_batch.params_grad_end_index, 5) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_first_diff_analyze.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_first_diff_analyze.py new file mode 100644 index 000000000..5c5a26907 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_first_diff_analyze.py @@ -0,0 +1,106 @@ +import unittest +from unittest.mock import patch + +import pandas as pd + +from msprobe.core.common.utils import CompareException +from msprobe.core.compare.diff_analyze.first_diff_analyze import FirstDiffAnalyze + + +class TestFirstDiffAnalyze(unittest.TestCase): + def setUp(self): + self.header = ['NPU name', 'L2norm diff', + 'MaxRelativeErr', 'MinRelativeErr', 'MeanRelativeErr', 'NormRelativeErr'] + self.data = [ + ['Functional.conv2d.0.forward.input.0', 1, '0.0%', '0.0%', '0.0%', '0.0%'], + ['Functional.conv2d.0.forward.input.1', 1, '99.0%', '99.0%', '99.0%', '99.0%'] + ] + self.result_df = pd.DataFrame(self.data, columns=self.header) + + @patch('msprobe.core.compare.diff_analyze.first_diff_analyze.thresholds', + {'compare_metrics': ['MaxRelativeErr', 'NormRelativeErr'], 'MaxRelativeErr': [0.5]}) + def test_single_metric_diff_check_true(self): + first_diff_analyze = FirstDiffAnalyze() + result = first_diff_analyze.single_metric_diff_check('MaxRelativeErr', '60.0%') + self.assertTrue(result) + + @patch('msprobe.core.compare.diff_analyze.first_diff_analyze.thresholds', + {'compare_metrics': ['MaxRelativeErr', 'NormRelativeErr'], 'MaxRelativeErr': [0.5]}) + def test_single_metric_diff_check_false(self): + first_diff_analyze = FirstDiffAnalyze() + result = first_diff_analyze.single_metric_diff_check('MaxRelativeErr', '30.0%') + self.assertFalse(result) + + @patch('msprobe.core.compare.diff_analyze.first_diff_analyze.thresholds', + {'compare_metrics': ['MaxRelativeErr', 'NormRelativeErr'], 'NormRelativeErr': [0.5]}) + def test_single_metric_diff_check_miss_threshold(self): + first_diff_analyze = FirstDiffAnalyze() + with self.assertRaises(CompareException) as context: + result = first_diff_analyze.single_metric_diff_check('MaxRelativeErr', '30.0%') + self.assertEqual(context.exception.code, CompareException.MISSING_THRESHOLD_ERROR) + + @patch('msprobe.core.compare.diff_analyze.first_diff_analyze.thresholds', + {'compare_metrics': ['MaxRelativeErr', 'NormRelativeErr'], 'MaxRelativeErr': [0.5, 1.0]}) + def test_single_metric_diff_check_wrong_threshold(self): + first_diff_analyze = FirstDiffAnalyze() + with self.assertRaises(CompareException) as context: + result = first_diff_analyze.single_metric_diff_check('MaxRelativeErr', '30.0%') + self.assertEqual(context.exception.code, CompareException.WRONG_THRESHOLD_ERROR) + + def test_single_api_check_within_threshold(self): + result_slice = [ + ['Functional.conv2d.0.forward.input.0', 1, '0.0%', '0.0%', '0.0%', '0.0%'], + ['Functional.conv2d.0.forward.input.1', 1, '0.1%', '0.1%', '0.1%', '0.1%'] + ] + expected_result = { + 'is_same': True, + 'op_items': [ + {'NPU name': 'Functional.conv2d.0.forward.input.0', 'L2norm diff': 1, + 'MaxRelativeErr': '0.0%', 'MinRelativeErr': '0.0%', + 'MeanRelativeErr': '0.0%', 'NormRelativeErr': '0.0%'}, + {'NPU name': 'Functional.conv2d.0.forward.input.1', 'L2norm diff': 1, + 'MaxRelativeErr': '0.1%', 'MinRelativeErr': '0.1%', + 'MeanRelativeErr': '0.1%', 'NormRelativeErr': '0.1%'} + ] + } + first_diff_analyze = FirstDiffAnalyze() + result = first_diff_analyze.single_api_check(result_slice, self.header) + self.assertEqual(result, expected_result) + + def test_single_api_check_exceed_threshold(self): + result_slice = [ + ['Functional.conv2d.0.forward.input.0', 1, '88.0%', '88.0%', '88.0%', '88.0%'], + ['Functional.conv2d.0.forward.input.1', 1, '99.0%', '99.0%', '99.0%', '99.0%'] + ] + expected_result = { + 'is_same': False, + 'op_items': [ + {'NPU name': 'Functional.conv2d.0.forward.input.0', 'L2norm diff': 1, + 'MaxRelativeErr': '88.0%', 'MinRelativeErr': '88.0%', + 'MeanRelativeErr': '88.0%', 'NormRelativeErr': '88.0%'}, + {'NPU name': 'Functional.conv2d.0.forward.input.1', 'L2norm diff': 1, + 'MaxRelativeErr': '99.0%', 'MinRelativeErr': '99.0%', + 'MeanRelativeErr': '99.0%', 'NormRelativeErr': '99.0%'}, + ] + } + first_diff_analyze = FirstDiffAnalyze() + result = first_diff_analyze.single_api_check(result_slice, self.header) + self.assertEqual(result, expected_result) + + def test_check(self): + expected_result = { + 'Functional.conv2d.0.forward': { + 'is_same': False, + 'op_items': [ + {'NPU name': 'Functional.conv2d.0.forward.input.0', 'L2norm diff': 1, + 'MaxRelativeErr': '0.0%', 'MinRelativeErr': '0.0%', + 'MeanRelativeErr': '0.0%', 'NormRelativeErr': '0.0%'}, + {'NPU name': 'Functional.conv2d.0.forward.input.1', 'L2norm diff': 1, + 'MaxRelativeErr': '99.0%', 'MinRelativeErr': '99.0%', + 'MeanRelativeErr': '99.0%', 'NormRelativeErr': '99.0%'}, + ] + } + } + first_diff_analyze = FirstDiffAnalyze() + result = first_diff_analyze.check(self.result_df) + self.assertEqual(result, expected_result) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_highlight.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_highlight.py index 933c84601..5a4ca7de4 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_highlight.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_highlight.py @@ -13,9 +13,10 @@ from openpyxl import load_workbook from openpyxl.styles import PatternFill from msprobe.core.common.const import CompareConst, Const -from msprobe.core.compare.highlight import ApiBatch, CheckMaxRelativeDiff, CheckOrderMagnitude, \ +from msprobe.core.compare.highlight import CheckMaxRelativeDiff, CheckOrderMagnitude, \ CheckOneThousandErrorRatio, CheckCosineSimilarity, add_highlight_row_info, HighLight from msprobe.core.compare.config import ModeConfig +from msprobe.core.compare.utils import ApiBatch summary_line_input = ['Functional_batch_norm_0_forward.input.0', 'Functional_batch_norm_0_forward.input.0', @@ -210,87 +211,6 @@ class TestUtilsMethods(unittest.TestCase): result = CheckMaxRelativeDiff().apply(info, color_columns, dump_mode=Const.SUMMARY) self.assertEqual(result, None) - def test_ApiBatch_increment_input(self): - api_name = "functional.conv2d" - start = 2 - api_batch = ApiBatch(api_name, start) - - api_batch.increment(Const.INPUT) - - self.assertEqual(api_batch._state, Const.INPUT) - self.assertEqual(api_batch.input_len, 2) - self.assertEqual(api_batch.params_end_index, 4) - self.assertEqual(api_batch.output_end_index, 4) - self.assertEqual(api_batch.params_grad_end_index, 4) - - def test_ApiBatch_increment_output(self): - api_name = "functional.conv2d" - start = 2 - api_batch = ApiBatch(api_name, start) - - api_batch.increment(Const.OUTPUT) - - self.assertEqual(api_batch._state, Const.OUTPUT) - self.assertEqual(api_batch.input_len, 1) - self.assertEqual(api_batch.params_end_index, 3) - self.assertEqual(api_batch.output_end_index, 4) - self.assertEqual(api_batch.params_grad_end_index, 4) - - def test_ApiBatch_increment_kwargs(self): - api_name = "functional.conv2d" - start = 2 - api_batch = ApiBatch(api_name, start) - - api_batch.increment(Const.KWARGS) - - self.assertEqual(api_batch._state, Const.KWARGS) - self.assertEqual(api_batch.input_len, 2) - self.assertEqual(api_batch.params_end_index, 4) - self.assertEqual(api_batch.output_end_index, 4) - self.assertEqual(api_batch.params_grad_end_index, 4) - - def test_ApiBatch_increment_params(self): - api_name = "functional.conv2d" - start = 2 - api_batch = ApiBatch(api_name, start) - - api_batch.increment(Const.PARAMS) - - self.assertEqual(api_batch._state, Const.PARAMS) - self.assertEqual(api_batch.input_len, 1) - self.assertEqual(api_batch.params_end_index, 4) - self.assertEqual(api_batch.output_end_index, 4) - self.assertEqual(api_batch.params_grad_end_index, 4) - - def test_ApiBatch_increment_multiple_input(self): - api_name = "functional.conv2d" - start = 2 - api_batch = ApiBatch(api_name, start) - - api_batch.increment(Const.INPUT) - api_batch.increment(Const.INPUT) - - self.assertEqual(api_batch._state, Const.INPUT) - self.assertEqual(api_batch.input_len, 3) - self.assertEqual(api_batch.params_end_index, 5) - self.assertEqual(api_batch.output_end_index, 5) - self.assertEqual(api_batch.params_grad_end_index, 5) - - def test_ApiBatch_increment_multiple_output(self): - api_name = "functional.conv2d" - start = 2 - api_batch = ApiBatch(api_name, start) - - api_batch.increment(Const.OUTPUT) - api_batch.increment(Const.OUTPUT) - - self.assertEqual(api_batch._state, Const.OUTPUT) - self.assertEqual(api_batch.input_len, 1) - self.assertEqual(api_batch.params_end_index, 3) - self.assertEqual(api_batch.output_end_index, 5) - self.assertEqual(api_batch.params_grad_end_index, 5) - - def test_find_error_rows_normal(self): compare_result = np.array([ ["Functional.linear.0.forward.input.0", "Functional.linear.0.forward.input.0", @@ -459,13 +379,6 @@ class TestUtilsMethods(unittest.TestCase): add_highlight_row_info(color_list, num, highlight_err_msg) self.assertEqual(color_list, [(1, ["a", "b"]), (5, ["c", "highlight"])]) - def test_add_highlight_row_info_new(self): - color_list = [(1, ["a", "b"]), (5, ["c"])] - num = 6 - highlight_err_msg = "highlight" - add_highlight_row_info(color_list, num, highlight_err_msg) - self.assertEqual(color_list, [(1, ["a", "b"]), (5, ["c"]), (6, ["highlight"])]) - def test_update_highlight_err_msg(self): data = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', 'torch.float32', 'torch.float32', [2, 2], [2, 2], -- Gitee From fd9a90991a382e07b174861ddde110e18419184d Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 18 Jul 2025 10:51:00 +0800 Subject: [PATCH 2/7] compare first_diff_analyze --- .../msprobe/core/compare/acc_compare.py | 4 ++-- .../diff_analyze/first_diff_analyze.py | 23 ++++++++++++++----- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index e439856c6..94bfc527e 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -113,9 +113,9 @@ class Comparator: return if self.mode_config.first_diff_analyze: - first_diff_analyze = FirstDiffAnalyze() + first_diff_analyze = FirstDiffAnalyze(self.mode_config) check_result = first_diff_analyze.check(result_df) - save_json(file_path, check_result) + save_json(file_path, check_result, indent=4) logger.info(f"Saving json file to disk: {file_path}") return diff --git a/debug/accuracy_tools/msprobe/core/compare/diff_analyze/first_diff_analyze.py b/debug/accuracy_tools/msprobe/core/compare/diff_analyze/first_diff_analyze.py index 00ecacc08..669ce71c8 100644 --- a/debug/accuracy_tools/msprobe/core/compare/diff_analyze/first_diff_analyze.py +++ b/debug/accuracy_tools/msprobe/core/compare/diff_analyze/first_diff_analyze.py @@ -15,9 +15,12 @@ import os +from msprobe.core.common.const import Const, CompareConst from msprobe.core.common.utils import safe_get_value, logger, CompareException from msprobe.core.common.file_utils import load_yaml -from msprobe.core.compare.utils import api_batches_update, get_name_and_state +from msprobe.core.compare.config import ModeConfig +from msprobe.core.compare.utils import api_batches_update + cur_dir = os.path.dirname(os.path.realpath(__file__)) diff_threshold_yaml_path = os.path.join(cur_dir, 'diff_analyze_threshold.yaml') @@ -26,6 +29,9 @@ cmp_metrics = thresholds.get('compare_metrics') class FirstDiffAnalyze: + def __init__(self, mode_config: ModeConfig): + self.mode_config = mode_config + @staticmethod def single_metric_diff_check(cmp_metric, metric_value): threshold = thresholds.get(cmp_metric, None) @@ -64,11 +70,16 @@ class FirstDiffAnalyze: } single_check_result['op_items'].append(op_item) - for cmp_metric in cmp_metrics: - metric_value = line[column_indices[cmp_metric]] - if self.single_metric_diff_check(cmp_metric, metric_value): + # set is_same + if self.mode_config.dump_mode == Const.MD5: + if line[column_indices[CompareConst.RESULT]] == CompareConst.DIFF: single_check_result['is_same'] = False - break + else: + for cmp_metric in cmp_metrics: + metric_value = line[column_indices[cmp_metric]] + if self.single_metric_diff_check(cmp_metric, metric_value): + single_check_result['is_same'] = False + break return single_check_result def check(self, result_df): @@ -105,4 +116,4 @@ class FirstDiffAnalyze: result_slice = result[api_batch.start: api_batch.params_grad_end_index] check_result[api_batch.api_name[: -1]] = self.single_api_check(result_slice, header) - return check_result \ No newline at end of file + return check_result -- Gitee From 61b563175c845c4a864df18292fd1e30fc663683 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 18 Jul 2025 14:24:48 +0800 Subject: [PATCH 3/7] compare first_diff_analyze --- .../core/compare/diff_analyze/first_diff_analyze.py | 10 +++------- debug/accuracy_tools/msprobe/core/compare/highlight.py | 10 +++------- debug/accuracy_tools/msprobe/core/compare/utils.py | 9 +++++++++ 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/diff_analyze/first_diff_analyze.py b/debug/accuracy_tools/msprobe/core/compare/diff_analyze/first_diff_analyze.py index 669ce71c8..badb4dedc 100644 --- a/debug/accuracy_tools/msprobe/core/compare/diff_analyze/first_diff_analyze.py +++ b/debug/accuracy_tools/msprobe/core/compare/diff_analyze/first_diff_analyze.py @@ -16,10 +16,10 @@ import os from msprobe.core.common.const import Const, CompareConst -from msprobe.core.common.utils import safe_get_value, logger, CompareException +from msprobe.core.common.utils import logger, CompareException from msprobe.core.common.file_utils import load_yaml from msprobe.core.compare.config import ModeConfig -from msprobe.core.compare.utils import api_batches_update +from msprobe.core.compare.utils import gen_api_batches cur_dir = os.path.dirname(os.path.realpath(__file__)) @@ -105,11 +105,7 @@ class FirstDiffAnalyze: result = result_df.values header = result_df.columns.tolist() - api_batches = [] - for i, res_i in enumerate(result): - api_full_name = safe_get_value(res_i, 0, "res_i") - api_name, state = get_name_and_state(api_full_name) - api_batches_update(api_batches, api_name, state, i) + api_batches = gen_api_batches(result) check_result = {} for api_batch in api_batches: diff --git a/debug/accuracy_tools/msprobe/core/compare/highlight.py b/debug/accuracy_tools/msprobe/core/compare/highlight.py index 6bdcec560..fb07b3d16 100644 --- a/debug/accuracy_tools/msprobe/core/compare/highlight.py +++ b/debug/accuracy_tools/msprobe/core/compare/highlight.py @@ -27,8 +27,8 @@ from tqdm import tqdm from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.file_utils import save_workbook from msprobe.core.common.log import logger -from msprobe.core.common.utils import get_header_index, safe_get_value -from msprobe.core.compare.utils import table_value_is_valid, api_batches_update +from msprobe.core.common.utils import get_header_index +from msprobe.core.compare.utils import table_value_is_valid, gen_api_batches from msprobe.core.compare.config import ModeConfig @@ -217,11 +217,7 @@ class HighLight: def find_compare_result_error_rows(self, result_df, highlight_dict): """将dataframe根据API分组，并找到有误差的算子用于高亮""" result = result_df.values - api_batches = [] - for i, res_i in enumerate(result): - api_name = safe_get_value(res_i, -1, "res_i") # 内部定义倒数第一个元素必是api_origin_name - state = safe_get_value(res_i, -2, "res_i") # 内部定义倒数第二个元素必是state - api_batches_update(api_batches, api_name, state, i) + api_batches = gen_api_batches(result) with tqdm(total=len(api_batches), desc="API/Module Analyse Progress", unit="item", ncols=100) as progress_bar: for api_batch in api_batches: self.find_error_rows(result[api_batch.start: api_batch.params_grad_end_index], api_batch, diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 4237b433d..faa8a17e3 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -586,6 +586,15 @@ def make_result_table(result, dump_mode, stack_mode): return result_df +def gen_api_batches(result: np.ndarray): + api_batches = [] + for i, res_i in enumerate(result): + api_name = safe_get_value(res_i, -1, "res_i") # 内部定义倒数第一个元素必是api_origin_name + state = safe_get_value(res_i, -2, "res_i") # 内部定义倒数第二个元素必是state + api_batches_update(api_batches, api_name, state, i) + return api_batches + + def _compare_parser(parser): parser.add_argument("-i", "--input_path", dest="input_path", type=str, help=" The compare input path, a dict json.", required=True) -- Gitee From cf5160e898f969de84594196d12eb0201d8b3612 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 18 Jul 2025 15:05:40 +0800 Subject: [PATCH 4/7] compare first_diff_analyze --- debug/accuracy_tools/msprobe/core/compare/utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index faa8a17e3..f7584990e 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -159,16 +159,19 @@ def is_leaf_data(op_data): def gen_op_item(op_data, op_name, state): op_item = {} op_item.update(op_data) + data_name = op_data.get(Const.DATA_NAME) if op_data.get(Const.DATA_NAME) else '-1' # 如果是""也返回-1 op_item[Const.DATA_NAME] = data_name op_item['full_op_name'] = data_name.rsplit(Const.SEP, 1)[0] if data_name != '-1' else op_name op_item[Const.STATE] = state + # 补齐统计量字段 params = [Const.MAX, Const.MIN, Const.MEAN, Const.NORM] for i in params: if i not in op_item: op_item[i] = None + # special cases if not op_item.get('dtype'): if op_item.get('type') == 'torch.Size': op_item['dtype'] = op_data.get('type') @@ -181,11 +184,18 @@ def gen_op_item(op_data, op_name, state): op_item['shape'] = '[]' for i in params: op_item[i] = op_data.get('value') + elif op_name.split(Const.SEP)[-1] in ['src', 'dst']: + op_item['dtype'] = op_data.get('type') + op_item['shape'] = '[]' + for i in params: + op_item[i] = str(op_data.get('value')) + op_item['md5'] = str(op_data.get('value')) elif op_item.get('type') == 'torch.ProcessGroup': op_item['dtype'] = op_data.get('type') op_item['shape'] = '[]' for i in params: op_item[i] = str(op_data.get('group_ranks')) + op_item['md5'] = str(op_data.get('group_ranks')) else: op_item['dtype'] = str(type(op_data.get('value'))) op_item['shape'] = '[]' -- Gitee From b1519ce1c67b34d09882e6c577d7e7966ea14b40 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 18 Jul 2025 16:39:11 +0800 Subject: [PATCH 5/7] compare first_diff_analyze --- debug/accuracy_tools/msprobe/core/compare/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index f7584990e..df8781f9f 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -184,7 +184,7 @@ def gen_op_item(op_data, op_name, state): op_item['shape'] = '[]' for i in params: op_item[i] = op_data.get('value') - elif op_name.split(Const.SEP)[-1] in ['src', 'dst']: + elif op_name.split(Const.SEP)[-1] in ['src', 'dst', 'group_src', 'group_dst']: op_item['dtype'] = op_data.get('type') op_item['shape'] = '[]' for i in params: -- Gitee From 2067ff8df730909c466057080ec862691e983232 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Sat, 19 Jul 2025 10:42:46 +0800 Subject: [PATCH 6/7] compare first_diff_analyze --- .../msprobe/core/compare/acc_compare.py | 2 + .../diff_analyze/first_diff_analyze.py | 4 +- .../msprobe/core/compare/utils.py | 3 + .../compare/test_cmp_first_diff_analyze.py | 106 ++++++++++++++---- 4 files changed, 92 insertions(+), 23 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index cef483e31..de30dba29 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -169,6 +169,8 @@ class Comparator: match_result.loc[~match.gen_dtype_condition(match_result), bench_columns] = CompareConst.N_A # organize compare result table by renaming columns + if self.mode_config.dump_mode == Const.ALL and self.mode_config.first_diff_analyze: + self.mode_config.dump_mode = Const.SUMMARY create_table = CreateTable(self.mode_config) result_df, header = create_table.make_result_df(match_result) diff --git a/debug/accuracy_tools/msprobe/core/compare/diff_analyze/first_diff_analyze.py b/debug/accuracy_tools/msprobe/core/compare/diff_analyze/first_diff_analyze.py index badb4dedc..f1192d895 100644 --- a/debug/accuracy_tools/msprobe/core/compare/diff_analyze/first_diff_analyze.py +++ b/debug/accuracy_tools/msprobe/core/compare/diff_analyze/first_diff_analyze.py @@ -84,7 +84,7 @@ class FirstDiffAnalyze: def check(self, result_df): """ - 比对后循环遍历api检查norm差异 + 比对后循环遍历api检查差异 example： { 'Functional.conv2d.0.forward': { @@ -110,6 +110,6 @@ class FirstDiffAnalyze: check_result = {} for api_batch in api_batches: result_slice = result[api_batch.start: api_batch.params_grad_end_index] - check_result[api_batch.api_name[: -1]] = self.single_api_check(result_slice, header) + check_result[api_batch.api_name] = self.single_api_check(result_slice, header) return check_result diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index d1b8d3031..8acbd0b12 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -629,6 +629,9 @@ def _compare_parser(parser): def compare_distributed_inner(npu_dump_dir, bench_dump_dir, output_path, compare_func, **kwargs): + if not isinstance(kwargs.get('first_diff_analyze', False), bool): + logger.error('kwargs: first_diff_analyze should be bool, please check!') + raise CompareException(CompareException.INVALID_PARAM_ERROR) if kwargs.get('suffix'): logger.error("Argument 'suffix' is not supported for compare_distributed.") raise CompareException(CompareException.INVALID_PARAM_ERROR) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_first_diff_analyze.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_first_diff_analyze.py index 5c5a26907..c7919efba 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_first_diff_analyze.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_first_diff_analyze.py @@ -3,38 +3,44 @@ from unittest.mock import patch import pandas as pd +from msprobe.core.common.const import Const, CompareConst from msprobe.core.common.utils import CompareException from msprobe.core.compare.diff_analyze.first_diff_analyze import FirstDiffAnalyze +from msprobe.core.compare.config import ModeConfig class TestFirstDiffAnalyze(unittest.TestCase): def setUp(self): self.header = ['NPU name', 'L2norm diff', - 'MaxRelativeErr', 'MinRelativeErr', 'MeanRelativeErr', 'NormRelativeErr'] + 'MaxRelativeErr', 'MinRelativeErr', 'MeanRelativeErr', 'NormRelativeErr', + 'state', 'api_origin_name'] self.data = [ - ['Functional.conv2d.0.forward.input.0', 1, '0.0%', '0.0%', '0.0%', '0.0%'], - ['Functional.conv2d.0.forward.input.1', 1, '99.0%', '99.0%', '99.0%', '99.0%'] + ['Functional.conv2d.0.forward.input.0', 1, '0.0%', '0.0%', '0.0%', '0.0%', 'input', 'Functional.conv2d.0.forward'], + ['Functional.conv2d.0.forward.input.1', 1, '99.0%', '99.0%', '99.0%', '99.0%', 'input', 'Functional.conv2d.0.forward'] ] self.result_df = pd.DataFrame(self.data, columns=self.header) @patch('msprobe.core.compare.diff_analyze.first_diff_analyze.thresholds', {'compare_metrics': ['MaxRelativeErr', 'NormRelativeErr'], 'MaxRelativeErr': [0.5]}) def test_single_metric_diff_check_true(self): - first_diff_analyze = FirstDiffAnalyze() + mode_config = ModeConfig(first_diff_analyze=True) + first_diff_analyze = FirstDiffAnalyze(mode_config) result = first_diff_analyze.single_metric_diff_check('MaxRelativeErr', '60.0%') self.assertTrue(result) @patch('msprobe.core.compare.diff_analyze.first_diff_analyze.thresholds', {'compare_metrics': ['MaxRelativeErr', 'NormRelativeErr'], 'MaxRelativeErr': [0.5]}) def test_single_metric_diff_check_false(self): - first_diff_analyze = FirstDiffAnalyze() + mode_config = ModeConfig(first_diff_analyze=True) + first_diff_analyze = FirstDiffAnalyze(mode_config) result = first_diff_analyze.single_metric_diff_check('MaxRelativeErr', '30.0%') self.assertFalse(result) @patch('msprobe.core.compare.diff_analyze.first_diff_analyze.thresholds', {'compare_metrics': ['MaxRelativeErr', 'NormRelativeErr'], 'NormRelativeErr': [0.5]}) def test_single_metric_diff_check_miss_threshold(self): - first_diff_analyze = FirstDiffAnalyze() + mode_config = ModeConfig(first_diff_analyze=True) + first_diff_analyze = FirstDiffAnalyze(mode_config) with self.assertRaises(CompareException) as context: result = first_diff_analyze.single_metric_diff_check('MaxRelativeErr', '30.0%') self.assertEqual(context.exception.code, CompareException.MISSING_THRESHOLD_ERROR) @@ -42,65 +48,123 @@ class TestFirstDiffAnalyze(unittest.TestCase): @patch('msprobe.core.compare.diff_analyze.first_diff_analyze.thresholds', {'compare_metrics': ['MaxRelativeErr', 'NormRelativeErr'], 'MaxRelativeErr': [0.5, 1.0]}) def test_single_metric_diff_check_wrong_threshold(self): - first_diff_analyze = FirstDiffAnalyze() + mode_config = ModeConfig(first_diff_analyze=True) + first_diff_analyze = FirstDiffAnalyze(mode_config) with self.assertRaises(CompareException) as context: result = first_diff_analyze.single_metric_diff_check('MaxRelativeErr', '30.0%') self.assertEqual(context.exception.code, CompareException.WRONG_THRESHOLD_ERROR) def test_single_api_check_within_threshold(self): result_slice = [ - ['Functional.conv2d.0.forward.input.0', 1, '0.0%', '0.0%', '0.0%', '0.0%'], - ['Functional.conv2d.0.forward.input.1', 1, '0.1%', '0.1%', '0.1%', '0.1%'] + ['Functional.conv2d.0.forward.input.0', 1, '0.0%', '0.0%', '0.0%', '0.0%', 'input', 'Functional.conv2d.0.forward'], + ['Functional.conv2d.0.forward.input.1', 1, '0.1%', '0.1%', '0.1%', '0.1%', 'input', 'Functional.conv2d.0.forward'] ] expected_result = { 'is_same': True, 'op_items': [ {'NPU name': 'Functional.conv2d.0.forward.input.0', 'L2norm diff': 1, 'MaxRelativeErr': '0.0%', 'MinRelativeErr': '0.0%', - 'MeanRelativeErr': '0.0%', 'NormRelativeErr': '0.0%'}, + 'MeanRelativeErr': '0.0%', 'NormRelativeErr': '0.0%', + 'state': 'input', 'api_origin_name': 'Functional.conv2d.0.forward'}, {'NPU name': 'Functional.conv2d.0.forward.input.1', 'L2norm diff': 1, 'MaxRelativeErr': '0.1%', 'MinRelativeErr': '0.1%', - 'MeanRelativeErr': '0.1%', 'NormRelativeErr': '0.1%'} + 'MeanRelativeErr': '0.1%', 'NormRelativeErr': '0.1%', + 'state': 'input', 'api_origin_name': 'Functional.conv2d.0.forward'} ] } - first_diff_analyze = FirstDiffAnalyze() + mode_config = ModeConfig(first_diff_analyze=True) + first_diff_analyze = FirstDiffAnalyze(mode_config) result = first_diff_analyze.single_api_check(result_slice, self.header) self.assertEqual(result, expected_result) def test_single_api_check_exceed_threshold(self): result_slice = [ - ['Functional.conv2d.0.forward.input.0', 1, '88.0%', '88.0%', '88.0%', '88.0%'], - ['Functional.conv2d.0.forward.input.1', 1, '99.0%', '99.0%', '99.0%', '99.0%'] + ['Functional.conv2d.0.forward.input.0', 1, '88.0%', '88.0%', '88.0%', '88.0%', 'input', 'Functional.conv2d.0.forward'], + ['Functional.conv2d.0.forward.input.1', 1, '99.0%', '99.0%', '99.0%', '99.0%', 'input', 'Functional.conv2d.0.forward'] ] expected_result = { 'is_same': False, 'op_items': [ {'NPU name': 'Functional.conv2d.0.forward.input.0', 'L2norm diff': 1, 'MaxRelativeErr': '88.0%', 'MinRelativeErr': '88.0%', - 'MeanRelativeErr': '88.0%', 'NormRelativeErr': '88.0%'}, + 'MeanRelativeErr': '88.0%', 'NormRelativeErr': '88.0%', + 'state': 'input', 'api_origin_name': 'Functional.conv2d.0.forward'}, {'NPU name': 'Functional.conv2d.0.forward.input.1', 'L2norm diff': 1, 'MaxRelativeErr': '99.0%', 'MinRelativeErr': '99.0%', - 'MeanRelativeErr': '99.0%', 'NormRelativeErr': '99.0%'}, + 'MeanRelativeErr': '99.0%', 'NormRelativeErr': '99.0%', + 'state': 'input', 'api_origin_name': 'Functional.conv2d.0.forward'}, ] } - first_diff_analyze = FirstDiffAnalyze() + mode_config = ModeConfig(first_diff_analyze=True) + first_diff_analyze = FirstDiffAnalyze(mode_config) result = first_diff_analyze.single_api_check(result_slice, self.header) self.assertEqual(result, expected_result) - def test_check(self): + def test_single_api_check_md5_same_true(self): + md5_header = CompareConst.MD5_COMPARE_RESULT_HEADER + [CompareConst.STACK, Const.STATE, Const.API_ORIGIN_NAME] + result_slice = [ + ['Functional.conv2d.0.forward.input.0', 'Functional.conv2d.0.forward.input.0', 'torch.int32', 'torch.int32', + '[]', '[]', '2144df1c', '2144df1c', 'pass', '', 'input', 'Functional.conv2d.0.forward'] + ] + expected_result = { + 'is_same': True, + 'op_items': [ + {CompareConst.NPU_NAME: 'Functional.conv2d.0.forward.input.0', + CompareConst.BENCH_NAME: 'Functional.conv2d.0.forward.input.0', + CompareConst.NPU_DTYPE: 'torch.int32', CompareConst.BENCH_DTYPE: 'torch.int32', + CompareConst.NPU_SHAPE: '[]', CompareConst.BENCH_SHAPE: '[]', + CompareConst.NPU_MD5: '2144df1c', CompareConst.BENCH_MD5: '2144df1c', + CompareConst.RESULT: 'pass', CompareConst.STACK: '', + Const.STATE: 'input', Const.API_ORIGIN_NAME: 'Functional.conv2d.0.forward' + } + ] + } + mode_config = ModeConfig(dump_mode=Const.MD5, first_diff_analyze=True) + first_diff_analyze = FirstDiffAnalyze(mode_config) + result = first_diff_analyze.single_api_check(result_slice, md5_header) + self.assertEqual(result, expected_result) + + def test_single_api_check_md5_same_false(self): + md5_header = CompareConst.MD5_COMPARE_RESULT_HEADER + [CompareConst.STACK, Const.STATE, Const.API_ORIGIN_NAME] + result_slice = [ + ['Functional.conv2d.0.forward.input.0', 'Functional.conv2d.0.forward.input.0', 'torch.int32', 'torch.int32', + '[]', '[]', '2144df1c', '2100df1c', 'Different', '', 'input', 'Functional.conv2d.0.forward'] + ] + expected_result = { + 'is_same': False, + 'op_items': [ + {CompareConst.NPU_NAME: 'Functional.conv2d.0.forward.input.0', + CompareConst.BENCH_NAME: 'Functional.conv2d.0.forward.input.0', + CompareConst.NPU_DTYPE: 'torch.int32', CompareConst.BENCH_DTYPE: 'torch.int32', + CompareConst.NPU_SHAPE: '[]', CompareConst.BENCH_SHAPE: '[]', + CompareConst.NPU_MD5: '2144df1c', CompareConst.BENCH_MD5: '2100df1c', + CompareConst.RESULT: 'Different', CompareConst.STACK: '', + Const.STATE: 'input', Const.API_ORIGIN_NAME: 'Functional.conv2d.0.forward' + } + ] + } + mode_config = ModeConfig(dump_mode=Const.MD5, first_diff_analyze=True) + first_diff_analyze = FirstDiffAnalyze(mode_config) + result = first_diff_analyze.single_api_check(result_slice, md5_header) + self.assertEqual(result, expected_result) + + def test_check_summary(self): expected_result = { 'Functional.conv2d.0.forward': { 'is_same': False, 'op_items': [ {'NPU name': 'Functional.conv2d.0.forward.input.0', 'L2norm diff': 1, 'MaxRelativeErr': '0.0%', 'MinRelativeErr': '0.0%', - 'MeanRelativeErr': '0.0%', 'NormRelativeErr': '0.0%'}, + 'MeanRelativeErr': '0.0%', 'NormRelativeErr': '0.0%', + 'state': 'input', 'api_origin_name': 'Functional.conv2d.0.forward'}, {'NPU name': 'Functional.conv2d.0.forward.input.1', 'L2norm diff': 1, 'MaxRelativeErr': '99.0%', 'MinRelativeErr': '99.0%', - 'MeanRelativeErr': '99.0%', 'NormRelativeErr': '99.0%'}, + 'MeanRelativeErr': '99.0%', 'NormRelativeErr': '99.0%', + 'state': 'input', 'api_origin_name': 'Functional.conv2d.0.forward'}, ] } } - first_diff_analyze = FirstDiffAnalyze() + mode_config = ModeConfig(first_diff_analyze=True) + first_diff_analyze = FirstDiffAnalyze(mode_config) result = first_diff_analyze.check(self.result_df) self.assertEqual(result, expected_result) -- Gitee From fbe5ab68864a045875a145b8951515aad56f6359 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 21 Jul 2025 09:46:03 +0800 Subject: [PATCH 7/7] compare first_diff_analyze --- debug/accuracy_tools/msprobe/core/compare/acc_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index de30dba29..af9a518ff 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -705,7 +705,7 @@ class CalcStatsDiff: # 相对误差转成百分比字符串 cond_ref_err = cond_not_nan_diff & ~condition_pt_zero result_df.loc[cond_ref_err, rel_err_name] = ( - result_df.loc[cond_ref_err, diff_name] / bench_val[cond_ref_err] * 100) + result_df.loc[cond_ref_err, diff_name] / bench_val[cond_ref_err].astype(float) * 100) result_df.loc[cond_ref_err, rel_err_name] = (result_df.loc[cond_ref_err, rel_err_name].abs().astype(str) + '%') magnitude = self.get_number(result_df[diff_name]).abs() / (pd.Series( -- Gitee