diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index dfa83c3d3a48ce0507ededd1451245559210ac83..1705129e7cfac7986b962ea77ae97e3fb7473e00 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -379,6 +379,9 @@ class Const: MATCH_MODE_NAME = "pure name" MATCH_MODE_MAPPING = "mapping" MATCH_MODE_SIMILARITY = "similarity" + CONFIG_CHECK_PASS = "pass" + CONFIG_CHECK_WARNING = "warning" + CONFIG_CHECK_ERROR = "error" class CompareConst: diff --git a/debug/accuracy_tools/msprobe/core/config_check/checkers/dataset_checker.py b/debug/accuracy_tools/msprobe/core/config_check/checkers/dataset_checker.py index f8eea02323be92385fdba87f9a456260b2453e79..aab5a7d267945930402f30cf856803a5558c8143 100644 --- a/debug/accuracy_tools/msprobe/core/config_check/checkers/dataset_checker.py +++ b/debug/accuracy_tools/msprobe/core/config_check/checkers/dataset_checker.py @@ -22,6 +22,7 @@ from msprobe.core.config_check.config_checker import register_checker_item, regi from msprobe.core.config_check.utils.utils import config_checking_print, get_tensor_features from msprobe.core.common.decorator import recursion_depth_decorator from msprobe.core.common.framework_adapter import FmkAdp +from msprobe.core.common.const import Const @recursion_depth_decorator("config_check: process_obj") @@ -134,5 +135,5 @@ class DatasetChecker(BaseChecker): cmp_dataset_pack_path = os.path.join(cmp_dir, DatasetChecker.target_name_in_zip) df = compare_dataset(bench_dataset_pack_path, cmp_dataset_pack_path) - pass_check = False not in df['equal'].values + pass_check = Const.CONFIG_CHECK_PASS if False not in df['equal'].values else Const.CONFIG_CHECK_ERROR return DatasetChecker.target_name_in_zip, pass_check, df diff --git a/debug/accuracy_tools/msprobe/core/config_check/checkers/env_args_checker.py b/debug/accuracy_tools/msprobe/core/config_check/checkers/env_args_checker.py index d4f72a6b26850322aa5c7685745cfe5b54bdb8a1..513a9f3b69c24225e4ba7a54f364104dce079dff 100644 --- a/debug/accuracy_tools/msprobe/core/config_check/checkers/env_args_checker.py +++ b/debug/accuracy_tools/msprobe/core/config_check/checkers/env_args_checker.py @@ -21,7 +21,7 @@ import pandas as pd from msprobe.core.common.file_utils import load_json, load_yaml, create_file_with_content, create_file_in_zip from msprobe.core.config_check.checkers.base_checker import BaseChecker from msprobe.core.config_check.config_checker import register_checker_item -from msprobe.core.config_check.utils.utils import config_checking_print +from msprobe.core.config_check.utils.utils import config_checking_print, process_pass_check from msprobe.core.common.const import Const @@ -59,17 +59,17 @@ def compare_env_data(npu_path, bench_path): cmp_env_name = cmp_env["name"] cmp_value = cmp_data.get(cmp_env_name, value[cmp_type]["default_value"]) if not bench_env: - data.append(["only cmp has this env", cmp_env["name"], "", cmp_value, "warning"]) + data.append(["only cmp has this env", cmp_env["name"], "", cmp_value, Const.CONFIG_CHECK_WARNING]) continue bench_env_name = bench_env["name"] bench_value = bench_data.get(bench_env_name, value[bench_type]["default_value"]) if cmp_value != bench_value: - data.append([bench_env_name, cmp_env_name, bench_value, cmp_value, "error"]) + data.append([bench_env_name, cmp_env_name, bench_value, cmp_value, Const.CONFIG_CHECK_ERROR]) else: bench_env_name = bench_env["name"] bench_value = bench_data.get(bench_env_name) if bench_data.get(bench_env_name) else value[bench_type][ "default_value"] - data.append([bench_env_name, "only bench has this env", bench_value, "", "warning"]) + data.append([bench_env_name, "only bench has this env", bench_value, "", Const.CONFIG_CHECK_WARNING]) df = pd.DataFrame(data, columns=EnvArgsChecker.result_header) return df @@ -92,5 +92,5 @@ class EnvArgsChecker(BaseChecker): bench_env_data = os.path.join(bench_dir, EnvArgsChecker.target_name_in_zip) cmp_env_data = os.path.join(cmp_dir, EnvArgsChecker.target_name_in_zip) df = compare_env_data(bench_env_data, cmp_env_data) - pass_check = "error" not in df['level'].values + pass_check = process_pass_check(df['level'].values) return EnvArgsChecker.target_name_in_zip, pass_check, df diff --git a/debug/accuracy_tools/msprobe/core/config_check/checkers/hyperparameter_checker.py b/debug/accuracy_tools/msprobe/core/config_check/checkers/hyperparameter_checker.py index 774abef4877786268bf700bbb695586800ef64d0..b1f1bda2810ecf7f0cc5234fb89801e8b14c10e1 100644 --- a/debug/accuracy_tools/msprobe/core/config_check/checkers/hyperparameter_checker.py +++ b/debug/accuracy_tools/msprobe/core/config_check/checkers/hyperparameter_checker.py @@ -22,7 +22,7 @@ import pandas as pd from msprobe.core.config_check.checkers.base_checker import BaseChecker from msprobe.core.config_check.config_checker import register_checker_item -from msprobe.core.config_check.utils.utils import compare_dict, config_checking_print, update_dict +from msprobe.core.config_check.utils.utils import compare_dict, config_checking_print, update_dict, process_pass_check from msprobe.core.config_check.utils.hyperparameter_parser import ParserFactory from msprobe.core.common.file_utils import (os_walk_for_files, create_file_in_zip, load_json, create_file_with_list, FileOpen, load_yaml) @@ -85,7 +85,7 @@ class HyperparameterChecker(BaseChecker): all_diffs.extend( HyperparameterChecker.compare_param(bench_hyperparameters, cmp_hyperparameters, file_name)) df = pd.DataFrame(all_diffs, columns=HyperparameterChecker.result_header) - pass_check = "error" not in df["level"].values + pass_check = process_pass_check(df["level"].values) return HyperparameterChecker.target_name_in_zip, pass_check, df @staticmethod @@ -101,13 +101,15 @@ class HyperparameterChecker(BaseChecker): if bench_param_value != cmp_param_value: all_diffs.append( [file_name, bench_param_name, matched_cmp_param_name, bench_param_value, cmp_param_value, - matched_with, "error"]) + matched_with, Const.CONFIG_CHECK_ERROR]) del cmp_params[matched_cmp_param_name] else: all_diffs.append( - [file_name, bench_param_name, "Only in benchmark", bench_param_value, "", "", "warning"]) + [file_name, bench_param_name, "Only in benchmark", bench_param_value, "", "", + Const.CONFIG_CHECK_WARNING]) for cmp_param_name, cmp_param_value in cmp_params.items(): - all_diffs.append([file_name, "Only in comparison", cmp_param_name, "", cmp_param_value, "", "warning"]) + all_diffs.append( + [file_name, "Only in comparison", cmp_param_name, "", cmp_param_value, "", Const.CONFIG_CHECK_WARNING]) all_diffs.sort() return all_diffs diff --git a/debug/accuracy_tools/msprobe/core/config_check/checkers/pip_checker.py b/debug/accuracy_tools/msprobe/core/config_check/checkers/pip_checker.py index a35bc3e00cd5bf5ed6601ce9983bad390f4b989f..0795ad6bc02d1418e2145a73b31cec219ff7dee4 100644 --- a/debug/accuracy_tools/msprobe/core/config_check/checkers/pip_checker.py +++ b/debug/accuracy_tools/msprobe/core/config_check/checkers/pip_checker.py @@ -23,8 +23,9 @@ except ImportError: from msprobe.core.common.file_utils import load_yaml, create_file_in_zip from msprobe.core.config_check.checkers.base_checker import BaseChecker from msprobe.core.config_check.config_checker import register_checker_item -from msprobe.core.config_check.utils.utils import config_checking_print +from msprobe.core.config_check.utils.utils import config_checking_print, process_pass_check from msprobe.core.common.file_utils import FileOpen, save_excel +from msprobe.core.common.const import Const dirpath = os.path.dirname(__file__) depend_path = os.path.join(dirpath, "../resource/dependency.yaml") @@ -62,7 +63,7 @@ def compare_pip_data(bench_pip_path, cmp_pip_path, fmk): if bench_version != cmp_version: data.append([package, bench_version if bench_version else 'None', cmp_version if cmp_version else 'None', - "error"]) + Const.CONFIG_CHECK_ERROR]) df = pd.DataFrame(data, columns=PipPackageChecker.result_header) return df @@ -86,5 +87,5 @@ class PipPackageChecker(BaseChecker): bench_pip_path = os.path.join(bench_dir, PipPackageChecker.target_name_in_zip) cmp_pip_path = os.path.join(cmp_dir, PipPackageChecker.target_name_in_zip) df = compare_pip_data(bench_pip_path, cmp_pip_path, fmk) - pass_check = "error" not in df['level'].values + pass_check = process_pass_check(df['level'].values) return PipPackageChecker.target_name_in_zip, pass_check, df diff --git a/debug/accuracy_tools/msprobe/core/config_check/checkers/random_checker.py b/debug/accuracy_tools/msprobe/core/config_check/checkers/random_checker.py index 1d1d0a7e79feb63116dc40c139b74c9d5778a8f0..54f22d62cb40d9e632f352f97cf104776d5fd4c4 100644 --- a/debug/accuracy_tools/msprobe/core/config_check/checkers/random_checker.py +++ b/debug/accuracy_tools/msprobe/core/config_check/checkers/random_checker.py @@ -16,11 +16,12 @@ import random from functools import wraps -from typing import Callable +from typing import Callable, List, Dict, Tuple, Optional import inspect import os import json from collections import defaultdict +import difflib import numpy as np import pandas as pd @@ -30,83 +31,227 @@ from msprobe.core.config_check.checkers.base_checker import BaseChecker from msprobe.core.config_check.utils.utils import config_checking_print from msprobe.core.common.framework_adapter import FmkAdp from msprobe.core.common.const import Const +from msprobe.core.common.log import logger +# 数据结构：{随机操作名字: [{count: 调用次数, stack: 调用栈列表}]} +random_op_stats = defaultdict(list) -random_log_dict = defaultdict(dict) +def get_call_stack(frame) -> List[str]: + """获取详细的调用栈信息，每个元素包含完整路径、行号、函数名和代码行""" + stack = [] + current_frame = frame.f_back # 跳过当前函数 -def load_json_files(directory): - json_data = {} - for file in os.listdir(directory): - file_path = os.path.join(directory, file) - if file.startswith('rank') and file.endswith('.json'): - json_data.update(load_json(file_path)) - return json_data - + while current_frame: + frame_info = inspect.getframeinfo(current_frame) + filename = os.path.abspath(frame_info.filename) + code_line = frame_info.code_context[0].strip() if frame_info.code_context else "" -def get_file_and_line(position): - parts = position.rsplit(':', 1) - if len(parts) == 2: - file_name = os.path.basename(parts[0]) - line_num = parts[1] - return f"{file_name}:{line_num}" - return position + # 格式化为详细的栈帧信息 + stack_entry = f"File {filename}, line {frame_info.lineno}, in {frame_info.function}, {code_line}" + stack.append(stack_entry) + current_frame = current_frame.f_back -def compare_json_files(bench_data, cmp_data): - results = [] - for op in set(bench_data) | set(cmp_data): - bench_records = bench_data.get(op, {}) - cmp_records = cmp_data.get(op, {}) - all_positions = set() - for position in set(bench_records) | set(cmp_records): - all_positions.add(get_file_and_line(position)) - - for position in all_positions: - bench_count = 0 - cmp_count = 0 - for original_position, count in bench_records.items(): - if get_file_and_line(original_position) == position: - bench_count += count - for original_position, count in cmp_records.items(): - if get_file_and_line(original_position) == position: - cmp_count += count - results.append([op, position, bench_count == cmp_count, bench_count, cmp_count]) - return results - - -def compare_random(bench_dir='bench', cmp_dir='cmp'): - bench_data = load_json_files(bench_dir) - cmp_data = load_json_files(cmp_dir) - results = compare_json_files(bench_data, cmp_data) - df = pd.DataFrame(results, columns=RandomChecker.result_header) - return df + # 反转堆栈以显示正确的调用顺序（栈底到栈顶） + return stack[::-1] def track_random_call(func: Callable, name: str): + """记录随机函数的调用信息""" + @wraps(func) def wrapper(*args, **kwargs): frame = inspect.currentframe() - caller_frame = frame.f_back - caller_info = inspect.getframeinfo(caller_frame) - location = f"{os.path.abspath(caller_info.filename)}:{caller_info.lineno}" - - global random_log_dict - random_log_dict.setdefault(name, {}) - random_log_dict[name][location] = random_log_dict[name].get(location, 0) + 1 - + stack = get_call_stack(frame) + + # 更新调用统计：操作名 -> [{count: 次数, stack: 调用栈列表}] + # 检查是否已有相同调用栈的记录 + for entry in random_op_stats[name]: + if entry['stack'] == stack: + entry['count'] += 1 + break + else: + # 新增调用栈记录 + random_op_stats[name].append({'count': 1, 'stack': stack}) + try: result = func(*args, **kwargs) return result except Exception as e: raise e finally: - del frame, caller_frame - + del frame + return wrapper +def load_stats_files(directory: str) -> Dict[str, Dict[str, List[Dict]]]: + """加载目录下所有统计文件并按rank组织数据""" + rank_data = {} + for file in os.listdir(directory): + file_path = os.path.join(directory, file) + if file.startswith('rank') and file.endswith('.json'): + rank = os.path.basename(file.split('.')[0])[4:] + if not rank or not rank.isdigit(): + logger.error(f"extract rank id from {file} failed") + raise ValueError + + # 加载并存储数据 + data = load_json(file_path) + rank_data[int(rank)] = data + + return rank_data + + +def stack_match(stack1: List[str], stack2: List[str], threshold: float = 0.8) -> bool: + """ + 比较两个调用栈是否相似，同时考虑路径、函数名和代码行（各占1/3），每一层的相似度阈值需要达到0.8 + + 参数: + - stack1: 第一个调用栈列表 + - stack2: 第二个调用栈列表 + - threshold: 相似度阈值，默认0.8 + + 返回: + - 两个调用栈是否相似的布尔值 + """ + if len(stack1) != len(stack2): + return False + + for frame1, frame2 in zip(stack1, stack2): + # 提取路径、函数名和代码行 + path1, func1, code1 = _parse_frame(frame1) + path2, func2, code2 = _parse_frame(frame2) + + # 计算相似度得分 (路径、函数名、代码行各占1/3权重) + path_score = _compare_path(path1, path2) + func_score = 1.0 if func1 == func2 else 0.0 + # 代码相似度 + code_score = difflib.SequenceMatcher(None, code1, code2).ratio() + + frame_score = (path_score + func_score + code_score) / 3.0 + if frame_score < threshold: + return False + + return True + + +def _parse_frame(frame: str) -> Tuple[str, str, str]: + """ + 解析栈帧字符串，提取路径、函数名和代码行 + + 参数: + - frame: 栈帧字符串。格式为"File {path}, line {line}, in {func}, {code}" + + 返回: + - path, func, code + """ + path = func = code = '' + stack_info = frame.split(' ') + if len(stack_info) > 6: + path = stack_info[1][:-1] + func = stack_info[5][:-1] + code = ' '.join(stack_info[6:]) + return path, func, code + + +def _compare_path(path1: str, path2: str) -> float: + """比较两个路径的相似度，只考虑文件名""" + if not path1 or not path2: + return 0.0 + + # 提取文件名（忽略目录路径） + file1 = os.path.basename(path1) + file2 = os.path.basename(path2) + + return 1.0 if file1 == file2 else 0.0 + + +def find_matching_stack(bench_stack: List[str], cmp_stacks: List[Dict]) -> Optional[Dict]: + """ + 查找匹配的调用栈 + + 参数: + - bench_stack: 基准侧的调用栈列表 + - cmp_stacks: 比较侧的调用栈条目列表，每个条目是{'count': 次数, 'stack': 调用栈列表} + + 返回: + - 匹配的调用栈条目或None + """ + for cmp_entry in cmp_stacks: + if stack_match(cmp_entry['stack'], bench_stack): + return cmp_entry + + return None + + +def stack_list_to_string(stack_list): + """ + 将调用栈列表转换为换行分隔的字符串 + 如果输入是特殊标记（如"no match stack"），则直接返回 + """ + if isinstance(stack_list, list): + return '\n'.join(stack_list) + return stack_list + + +def compare_random_calls(bench_dir: str = 'bench', cmp_dir: str = 'cmp') -> pd.DataFrame: + """比较两个目录下的随机调用栈统计，生成详细比对结果""" + bench_rank_data = load_stats_files(bench_dir) + cmp_rank_data = load_stats_files(cmp_dir) + + # 获取所有rank + all_ranks = sorted(set(bench_rank_data.keys()) | set(cmp_rank_data.keys())) + + results = [] + + for rank in all_ranks: + bench_data = bench_rank_data.get(rank, {}) + cmp_data = cmp_rank_data.get(rank, {}) + + # 获取所有操作 + all_ops = set(bench_data.keys()) | set(cmp_data.keys()) + + for op in all_ops: + bench_stacks = bench_data.get(op, []) + cmp_stacks = cmp_data.get(op, []) + + # 处理bench侧的每个调用栈 + for bench_entry in bench_stacks: + bench_stack = bench_entry['stack'] + bench_count = bench_entry['count'] + + # 查找匹配的cmp侧调用栈 + cmp_entry = find_matching_stack(bench_stack, cmp_stacks) + + if cmp_entry: + cmp_count = cmp_entry['count'] + check_result = bench_count == cmp_count + results.append([op, rank, bench_stack, cmp_entry['stack'], bench_count, cmp_count, check_result]) + else: + # 没有匹配的调用栈 + results.append([op, rank, bench_stack, "no match stack", bench_count, 0, False]) + + # 处理cmp侧中没有在bench侧出现的调用栈 + for cmp_entry in cmp_stacks: + cmp_stack = cmp_entry['stack'] + # 检查是否已经在上面处理过 + if not any(stack_match(bench_entry['stack'], cmp_stack) for bench_entry in bench_stacks): + results.append([op, rank, "no match stack", cmp_stack, 0, cmp_entry['count'], False]) + + # 创建DataFrame + df = pd.DataFrame(results, columns=RandomChecker.result_header) + + # 应用转换函数 + df['bench_stack'] = df['bench_stack'].apply(stack_list_to_string) + df['cmp_stack'] = df['cmp_stack'].apply(stack_list_to_string) + + return df + + def torch_patchs(): + """补丁Torch随机函数""" import torch torch_patches = { 'rand': torch.rand, @@ -131,6 +276,7 @@ def torch_patchs(): def mindspore_patchs(): + """补丁MindSpore随机函数""" import mindspore mindspore_ops_patches = { @@ -146,42 +292,53 @@ def mindspore_patchs(): } for name, func in mindspore_patches.items(): setattr(mindspore, name, track_random_call(func, f"mindspore.{name}")) - + @register_checker_item("random") class RandomChecker(BaseChecker): input_needed = None - target_name_in_zip = "random" - result_header = ['op', 'position', 'equal', 'bench_count', 'cmp_count'] + result_header = ['op', 'rank', 'bench_stack', 'cmp_stack', 'bench_count', 'cmp_count', 'check_result'] write_once = False @staticmethod def pack(pack_input): + """打包随机调用统计到zip文件""" output_zip_path = pack_input.output_zip_path def collect_input(model, args, kwargs, step): if RandomChecker.write_once: return - random_log_filepath = os.path.join(RandomChecker.target_name_in_zip, f"rank{FmkAdp.get_rank_id()}.json") - create_file_in_zip(output_zip_path, random_log_filepath, json.dumps(random_log_dict, indent=4)) - config_checking_print(f"add first random_log input features to zip") + random_stats_dir = os.path.join(RandomChecker.target_name_in_zip) + stats_filepath = os.path.join(random_stats_dir, f"rank{FmkAdp.get_rank_id()}.json") + + # 转换为JSON格式：{操作名: [{count: 次数, stack: 调用栈列表}]} + stats_json = {} + for op_name, entries in random_op_stats.items(): + stats_json[op_name] = entries + + create_file_in_zip(output_zip_path, stats_filepath, json.dumps(stats_json, indent=4)) + config_checking_print(f"已将随机调用统计打包到: {stats_filepath}") RandomChecker.write_once = True - + register_pre_forward_fun_list(collect_input) @staticmethod def compare(bench_dir, cmp_dir, output_path, fmk): - bench_random_log_pack_path = os.path.join(bench_dir, RandomChecker.target_name_in_zip) - cmp_random_log_pack_path = os.path.join(cmp_dir, RandomChecker.target_name_in_zip) + """比较两组随机调用统计""" + bench_stats_path = os.path.join(bench_dir, RandomChecker.target_name_in_zip) + cmp_stats_path = os.path.join(cmp_dir, RandomChecker.target_name_in_zip) + + df = compare_random_calls(bench_stats_path, cmp_stats_path) + pass_check = Const.CONFIG_CHECK_PASS if False not in df['check_result'].values else Const.CONFIG_CHECK_ERROR - df = compare_random(bench_random_log_pack_path, cmp_random_log_pack_path) - pass_check = False not in df['equal'].values return RandomChecker.target_name_in_zip, pass_check, df @staticmethod def apply_patches(fmk=Const.PT_FRAMEWORK): + """应用随机函数补丁""" + # 补丁Python random模块 random_patches = { 'random': random.random, 'randint': random.randint, @@ -191,6 +348,7 @@ class RandomChecker(BaseChecker): for name, func in random_patches.items(): setattr(random, name, track_random_call(func, f"random.{name}")) + # 补丁Numpy随机函数 np_random_patches = { 'rand': np.random.rand, 'randint': np.random.randint, @@ -200,9 +358,10 @@ class RandomChecker(BaseChecker): for name, func in np_random_patches.items(): setattr(np.random, name, track_random_call(func, f"np.random.{name}")) + # 补丁框架特定随机函数 if fmk == Const.PT_FRAMEWORK: torch_patchs() elif fmk == Const.MS_FRAMEWORK: mindspore_patchs() else: - raise Exception(f"apply patches framework error, not in {FmkAdp.supported_fmk}") + raise Exception(f"不支持的框架: {fmk}, 支持的框架: {FmkAdp.supported_fmk}") diff --git a/debug/accuracy_tools/msprobe/core/config_check/checkers/weights_checker.py b/debug/accuracy_tools/msprobe/core/config_check/checkers/weights_checker.py index f17c62ff9271fe2bc95207cdf405071cb8289f80..32716ea2e86ab79b290b48f8f96a4e1fe862f4b5 100644 --- a/debug/accuracy_tools/msprobe/core/config_check/checkers/weights_checker.py +++ b/debug/accuracy_tools/msprobe/core/config_check/checkers/weights_checker.py @@ -22,6 +22,7 @@ from msprobe.core.config_check.checkers.base_checker import BaseChecker from msprobe.core.config_check.config_checker import register_checker_item, register_pre_forward_fun_list from msprobe.core.config_check.utils.utils import config_checking_print, get_tensor_features from msprobe.core.common.framework_adapter import FmkAdp +from msprobe.core.common.const import Const def collect_weights_data(model): @@ -143,5 +144,5 @@ class WeightsChecker(BaseChecker): bench_weight_pack_path = os.path.join(bench_dir, WeightsChecker.target_name_in_zip) cmp_weight_pack_path = os.path.join(cmp_dir, WeightsChecker.target_name_in_zip) df = compare_weight(bench_weight_pack_path, cmp_weight_pack_path) - pass_check = False not in df['equal'].values + pass_check = Const.CONFIG_CHECK_PASS if False not in df['equal'].values else Const.CONFIG_CHECK_ERROR return WeightsChecker.target_name_in_zip, pass_check, df diff --git a/debug/accuracy_tools/msprobe/core/config_check/utils/utils.py b/debug/accuracy_tools/msprobe/core/config_check/utils/utils.py index 8c3c329cf20e2b6fb890437b3ba9950f14cc8878..29ec6375b3ef8bea485b218bbe76cb2dc2567d35 100644 --- a/debug/accuracy_tools/msprobe/core/config_check/utils/utils.py +++ b/debug/accuracy_tools/msprobe/core/config_check/utils/utils.py @@ -19,6 +19,7 @@ import hashlib from msprobe.core.common.framework_adapter import FmkAdp from msprobe.core.common.log import logger +from msprobe.core.common.const import Const def merge_keys(dir_0, dir_1): @@ -105,3 +106,12 @@ def update_dict(ori_dict, new_dict): ori_dict[key] = {"description": "duplicate_value", "values": [ori_dict[key], new_dict[key]]} else: ori_dict[key] = value + + +def process_pass_check(data): + if Const.CONFIG_CHECK_ERROR in data: + return Const.CONFIG_CHECK_ERROR + elif Const.CONFIG_CHECK_WARNING in data: + return Const.CONFIG_CHECK_WARNING + else: + return Const.CONFIG_CHECK_PASS diff --git a/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_config_check.py b/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_config_check.py index 0a39432fa4161957a036e6394c42c81145052896..93e4a11bebafdbf663c70ce3f4a2476d72f05340 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_config_check.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_config_check.py @@ -17,11 +17,11 @@ from msprobe.core.config_check.checkers.weights_checker import WeightsChecker from msprobe.core.common.file_utils import read_xlsx from msprobe.core.common.framework_adapter import FmkAdp - testdir = os.path.dirname(__file__) config_checking_dir = os.path.dirname(testdir) temp_dir = os.path.join(config_checking_dir, "temp") os.makedirs(temp_dir, exist_ok=True) +ms.set_context(device_target="CPU") def seed_all(seed=1234, mode=False): @@ -110,18 +110,15 @@ def train_test(seed, output_zip_path, shell_path, mock_env, mock_pip): optimizer = ms_nn.SGD(test_module.trainable_params(), learning_rate=1e-2) train_network = ms_nn.TrainOneStepCell(ms_nn.WithLossCell(test_module, loss_fun), optimizer) ConfigChecker(test_module, shell_path, output_zip_path, fmk="mindspore") - + for input_data, label in ms_test_dataset: loss = train_network(input_data, label) - class TestConfigChecker(unittest.TestCase): def tearDown(self): FmkAdp.set_fmk("pytorch") - return shutil.rmtree(temp_dir) - def test_all(self): train_test(1234, os.path.join(temp_dir, "config_check_pack1.zip"), [os.path.join(testdir, "cmp.sh")]) @@ -143,12 +140,12 @@ class TestConfigChecker(unittest.TestCase): total_check_result = read_xlsx(os.path.join(compare_output_dir, ConfigChecker.result_filename)) self.assertEqual(total_check_result.columns.tolist(), ConfigChecker.result_header) target_total_check_result = [ - ['env', False], - ['pip', False], - ['dataset', False], - ['weights', False], - ['hyperparameters', False], - ['random', False] + ['env', "error"], + ['pip', "error"], + ['dataset', "error"], + ['weights', "error"], + ['hyperparameters', "error"], + ['random', "error"] ] self.assertEqual(total_check_result.values.tolist(), target_total_check_result) @@ -160,7 +157,7 @@ class TestConfigChecker(unittest.TestCase): random_check_result = read_xlsx(os.path.join(compare_output_dir, ConfigChecker.result_filename), sheet_name=RandomChecker.target_name_in_zip) self.assertEqual(random_check_result.columns.tolist(), RandomChecker.result_header) - self.assertEqual(len(random_check_result), 5) + self.assertEqual(len(random_check_result), 7) dataset_check_result = read_xlsx(os.path.join(compare_output_dir, ConfigChecker.result_filename), sheet_name=DatasetChecker.target_name_in_zip) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_random_checker.py b/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_random_checker.py index 9a6bdb89f83d0c154c86d6522ffc76bb042cff4d..6a4509480f5b629d77e3b4d36d7ffc2fc11eeb80 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_random_checker.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_random_checker.py @@ -1,77 +1,73 @@ import unittest -import pandas as pd -from unittest.mock import patch, MagicMock -from msprobe.core.config_check.checkers.random_checker import compare_json_files, compare_random, get_file_and_line +from msprobe.core.config_check.checkers.random_checker import stack_match -class TestCompareRandom(unittest.TestCase): +class TestStackMatch(unittest.TestCase): + def test_identical_stacks(self): + stack1 = [ + "File /project/utils/funcs.py, line 42, in calculate_sum, return a + b", + "File /project/main.py, line 15, in main, result = calculate_sum(1, 2)" + ] + stack2 = [ + "File /project/utils/funcs.py, line 42, in calculate_sum, return a + b", + "File /project/main.py, line 15, in main, result = calculate_sum(1, 2)" + ] + self.assertTrue(stack_match(stack1, stack2)) - @patch('os.listdir', return_value=['rank1.json', 'rank2.json']) - @patch('os.path.join', return_value='test_path') - @patch("msprobe.core.config_check.checkers.random_checker.load_json") - def test_compare_random_with_files(self, mock_load_json, mock_path, mock_listdir): - mock_load_json.return_value = {"op1": {"position1": 1}} - bench_dir = 'test_bench' - cmp_dir = 'test_cmp' - result = compare_random(bench_dir, cmp_dir) - self.assertEqual(isinstance(result, pd.DataFrame), True) + def test_different_paths_same_file(self): + stack1 = [ + "File /user1/project/utils/funcs.py, line 42, in calculate_sum, return a + b" + ] + stack2 = [ + "File /user2/another_project/utils/funcs.py, line 42, in calculate_sum, return a + b" + ] + # 文件名相同，函数名和代码行相同 + self.assertTrue(stack_match(stack1, stack2)) - @patch('os.listdir', return_value=[]) - @patch('os.path.join', return_value='test_path') - def test_compare_random_no_files(self, mock_path, mock_listdir): - bench_dir = 'test_bench' - cmp_dir = 'test_cmp' - result = compare_random(bench_dir, cmp_dir) - self.assertEqual(isinstance(result, pd.DataFrame), True) - self.assertEqual(len(result), 0) + def test_different_filenames(self): + stack1 = [ + "File /project/utils/funcs.py, line 42, in calculate_sum, return a + b" + ] + stack2 = [ + "File /project/utils/other_funcs.py, line 42, in calculate_sum, return a + b" + ] + # 文件名不同 + self.assertFalse(stack_match(stack1, stack2)) - def test_get_file_and_line_with_valid_input(self): - position = '/path/to/file.py:10' - result = get_file_and_line(position) - self.assertEqual(isinstance(result, str), True) - self.assertEqual(result, 'file.py:10') + def test_different_line_numbers(self): + stack1 = [ + "File /project/utils/funcs.py, line 42, in calculate_sum, return a + b" + ] + stack2 = [ + "File /project/utils/funcs.py, line 45, in calculate_sum, return a + b" + ] + self.assertTrue(stack_match(stack1, stack2)) - def test_get_file_and_line_with_invalid_input(self): - position = 'invalid_position' - result = get_file_and_line(position) - self.assertEqual(isinstance(result, str), True) - self.assertEqual(result, 'invalid_position') + def test_different_functions(self): + stack1 = [ + "File /project/utils/funcs.py, line 42, in calculate_sum, return a + b" + ] + stack2 = [ + "File /project/utils/funcs.py, line 42, in multiply, return a * b" + ] + self.assertFalse(stack_match(stack1, stack2)) - @patch('os.listdir', return_value=['rank1.json', 'rank2.json']) - @patch('os.path.join', return_value='test_path') - def test_compare_json_files_same_data(self, mock_path, mock_listdir): - bench_data = {"op1": {"position1:10": 1}} - cmp_data = {"op1": {"position1:10": 1}} - result = compare_json_files(bench_data, cmp_data) - self.assertEqual(isinstance(result, list), True) - self.assertEqual(len(result), 1) - self.assertEqual(result[0][2], True) + def test_similar_code_different_variables(self): + stack1 = [ + "File /project/main.py, line 15, in main, result = calculate_sum(a, b)" + ] + stack2 = [ + "File /project/main.py, line 15, in main, result = calculate_sum(x, y)" + ] + # 代码行前缀和结构相似 + self.assertTrue(stack_match(stack1, stack2)) - @patch('os.listdir', return_value=['rank1.json', 'rank2.json']) - @patch('os.path.join', return_value='test_path') - def test_compare_json_files_different_data(self, mock_path, mock_listdir): - bench_data = {"op1": {"position1:10": 1}} - cmp_data = {"op1": {"position1:10": 2}} - result = compare_json_files(bench_data, cmp_data) - self.assertEqual(isinstance(result, list), True) - self.assertEqual(len(result), 1) - self.assertEqual(result[0][2], False) - - @patch('os.listdir', return_value=['rank1.json', 'rank2.json']) - @patch('os.path.join', return_value='test_path') - def test_compare_json_files_missing_op_in_bench(self, mock_path, mock_listdir): - bench_data = {} - cmp_data = {"op1": {"position1:10": 1}} - result = compare_json_files(bench_data, cmp_data) - self.assertEqual(isinstance(result, list), True) - self.assertEqual(len(result), 1) - - @patch('os.listdir', return_value=['rank1.json', 'rank2.json']) - @patch('os.path.join', return_value='test_path') - def test_compare_json_files_missing_op_in_cmp(self, mock_path, mock_listdir): - bench_data = {"op1": {"position1:10": 1}} - cmp_data = {} - result = compare_json_files(bench_data, cmp_data) - self.assertEqual(isinstance(result, list), True) - self.assertEqual(len(result), 1) + def test_different_code_structure(self): + stack1 = [ + "File /project/main.py, line 15, in main, result = calculate_sum(a, b)" + ] + stack2 = [ + "File /project/main.py, line 15, in main, print('Hello, world!')" + ] + self.assertFalse(stack_match(stack1, stack2))