diff --git a/debug/accuracy_tools/api_checker/api_mapping.json b/debug/accuracy_tools/api_checker/api_mapping.json new file mode 100644 index 0000000000000000000000000000000000000000..f41b2b0f8d00ca603c55565ee0e8d853663e0485 --- /dev/null +++ b/debug/accuracy_tools/api_checker/api_mapping.json @@ -0,0 +1,23 @@ +{ + "Sqrt": "sqrt", + "Square": "square", + "Relu": "relu", + "Mul": "mul", + "Reshape": "reshape", + "ExpandDims": "expanddims", + "IsFinite": "isfinite", + "Cos": "cos", + "Transponse": "permute", + "ReadDiv": "true_divide", + "FloorDiv": "floor_divide", + "Neg": "neg", + "Reciprocal": "reciprocal", + "LogicalNot": "logical_not", + "Ceil": "ceil", + "Exp": "exp", + "Pow": "pow", + "Log": "log", + "Select": "where", + "Tile": "tile", + "Rsqrt": "rsqrt" +} \ No newline at end of file diff --git a/debug/accuracy_tools/api_checker/common/logger.py b/debug/accuracy_tools/api_checker/common/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..5d5016af72718a32cface7ece7b2c7e5348fc687 --- /dev/null +++ b/debug/accuracy_tools/api_checker/common/logger.py @@ -0,0 +1,30 @@ +import logging +from datetime import datetime + +class SingletonLogger: + _instance = None + + def __new__(cls): + if cls._instance is None: + cls._instance = super(SingletonLogger, cls).__new__(cls) + cls._instance._initialize_logger() + return cls._instance + + def _initialize_logger(self): + self.logger = logging.getLogger("singleton_logger") + handler = logging.StreamHandler() + handler.setFormatter(CustomFormatter()) + self.logger.addHandler(handler) + self.logger.setLevel(logging.DEBUG) + + def get_logger(self): + return self.logger + +class CustomFormatter(logging.Formatter): + def format(self, record): + level = record.levelname + time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + message = record.getMessage() + return f"[{level}] {time} - {message}" + +logger = SingletonLogger().get_logger() \ No newline at end of file diff --git a/debug/accuracy_tools/api_checker/common/utils.py b/debug/accuracy_tools/api_checker/common/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..428a47ebd9e4fba6b243301361f064ffeefbbfdb --- /dev/null +++ b/debug/accuracy_tools/api_checker/common/utils.py @@ -0,0 +1,203 @@ +import os +import json +import csv +import sys +import time +import stat +import inspect + +import numpy as np +import torch + +class Const: + """ + Class for const + """ + DIRECTORY_LENGTH = 4096 + FILE_NAME_LENGTH = 255 + FILE_PATTERN = r'^[a-zA-Z0-9_./-]+$' + MODEL_TYPE = ['.onnx', '.pb', '.om'] + SEMICOLON = ";" + COLON = ":" + EQUAL = "=" + COMMA = "," + DOT = "." + DUMP_RATIO_MAX = 100 + SUMMERY_DATA_NUMS = 256 + ONE_HUNDRED_MB = 100 * 1024 * 1024 + FLOAT_EPSILON = np.finfo(float).eps + SUPPORT_DUMP_MODE = ['api', 'acl'] + ON = 'ON' + OFF = 'OFF' + BACKWARD = 'backward' + FORWARD = 'forward' + FLOAT_TYPE = [np.half, np.single, float, np.double, np.float64, np.longdouble, np.float32, np.float16] + BOOL_TYPE = [bool, np.uint8] + INT_TYPE = [np.int32, np.int64] + + INPUT = "input" + OUTPUT = "output" + + # dump mode + ALL = "all" + LIST = "list" + RANGE = "range" + STACK = "stack" + ACL = "acl" + API_LIST = "api_list" + API_STACK = "api_stack" + DUMP_MODE = [ALL, LIST, RANGE, STACK, ACL, API_LIST, API_STACK] + + WRITE_FLAGS = os.O_WRONLY | os.O_CREAT + WRITE_MODES = stat.S_IWUSR | stat.S_IRUSR + + RAISE_PRECISION = { + torch.float16: torch.float32, + torch.bfloat16: torch.float32, + torch.float32: torch.float64 + } + CONVERT = { + "int32_to_int64": ["torch.int32", "torch.int64"], + } + + CONVERT_API = { + "int32_to_int64": ["cross_entropy"] + } + + +def _print_log(level, msg): + current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))) + pid = os.getgid() + print(current_time + "(" + str(pid) + ")-[" + level + "]" + msg) + sys.stdout.flush() + +def print_info_log(info_msg): + """ + Function Description: + print info log. + Parameter: + info_msg: the info message. + """ + _print_log("INFO", info_msg) + + +def print_error_log(error_msg): + """ + Function Description: + print error log. + Parameter: + error_msg: the error message. + """ + _print_log("ERROR", error_msg) + +def print_warn_log(warn_msg): + """ + Function Description: + print warn log. + Parameter: + warn_msg: the warning message. + """ + _print_log("WARNING", warn_msg) + + +def get_json_contents(file_path): + with open(file_path, "r") as f: + ops = f.read() + #ops = get_file_content_bytes(file_path) + try: + json_obj = json.loads(ops) + except ValueError as error: + print_error_log('Failed to load "%s". %s' % (file_path, str(error))) + #raise CompareException(CompareException.INVALID_FILE_ERROR) from error + if not isinstance(json_obj, dict): + print_error_log('Json file %s, content is not a dictionary!' % file_path) + #raise CompareException(CompareException.INVALID_FILE_ERROR) + return json_obj + + +def write_csv(data, filepath): + with open(filepath, 'a', encoding='utf-8-sig') as f: + writer = csv.writer(f) + writer.writerow(data) + + +def check_file_or_directory_path(path, isdir=False): + """ + Function Description: + check whether the path is valid + Parameter: + path: the path to check + isdir: the path is dir or file + Exception Description: + when invalid data throw exception + """ + if isdir: + if not os.path.exists(path): + print_error_log('The path {} is not exist.'.format(path)) + #raise CompareException(CompareException.INVALID_PATH_ERROR) + + if not os.path.isdir(path): + print_error_log('The path {} is not a directory.'.format(path)) + #raise CompareException(CompareException.INVALID_PATH_ERROR) + + if not os.access(path, os.W_OK): + print_error_log( + 'The path {} does not have permission to write. Please check the path permission'.format(path)) + #raise CompareException(CompareException.INVALID_PATH_ERROR) + else: + if not os.path.isfile(path): + print_error_log('{} is an invalid file or non-exist.'.format(path)) + #raise CompareException(CompareException.INVALID_PATH_ERROR) + + if not os.access(path, os.R_OK): + print_error_log( + 'The path {} does not have permission to read. Please check the path permission'.format(path)) + #raise CompareException(CompareException.INVALID_PATH_ERROR) + + +def get_stack(): + stack_str = [] + try: + for (_, path, line, func, code, _) in inspect.stack()[3:]: + if code: + stack_line = [path, str(line), func, code[0].strip() if code else code] + else: + stack_line = [path, str(line), func, code] + stack_str.append(stack_line) + except Exception as e: + print("Dump stack info failed, error: {}".format(e)) + stack_str.append('') + return stack_str + + +dtype_map = { + "Float32": np.float32, + "Float16": np.float16, + "Float64": np.float64, + "Int8": np.int8, + "Int16": np.int16, + "Int32": np.int32, + "Int64": np.int64, + "Bool_": np.bool_, + "Uint8": np.uint8, + "Uint16": np.uint16, + "Uint32": np.uint32, + "Uint64": np.uint64, + "Bool": np.bool_, + "Complex64": np.complex64, + "Complex128": np.complex128 + +} + + +np_scalar_type = [ + bool, + np.int8, + np.int16, + np.int32, + np.int64, + np.uint8, + np.uint16, + np.uint32, + np.uint64, +] \ No newline at end of file diff --git a/debug/accuracy_tools/api_checker/compare/algorithm.py b/debug/accuracy_tools/api_checker/compare/algorithm.py new file mode 100644 index 0000000000000000000000000000000000000000..2396001befd73210cdba54f9f4b2832f58335a77 --- /dev/null +++ b/debug/accuracy_tools/api_checker/compare/algorithm.py @@ -0,0 +1,229 @@ +import numpy as np +import torch + +from compare.compare_utils import CompareConst, precision_configs +from common.utils import Const + +FLOAT_EPSILON = np.finfo(float).eps +np.seterr(divide='ignore', invalid='ignore') # ignore `invalid value encountered in true_divide` warning +print("FLOAT_EPSILON:", FLOAT_EPSILON) +NAN = 'NaN' + +#cos +def cosine_sim(bench_output, device_output): + msg = "" + n_value = device_output.reshape(-1) + b_value = bench_output.reshape(-1) + cos = CompareConst.NA + np.seterr(divide="ignore", invalid="ignore") + if n_value.shape != b_value.shape: + msg = f"Shape of device and bench outputs don't match. device: {n_value.shape}, bench: {b_value.shape}." + return -1, False, msg + if len(n_value) == 1: + msg = "All the data in device dump data is scalar. Please refer to other compare algorithms." + return cos, True, msg + n_value_max = np.max(np.abs(n_value)) + b_value_max = np.max(np.abs(b_value)) + if n_value_max <= np.finfo(float).eps and b_value_max <= np.finfo(float).eps: + return cos, True, msg + elif n_value_max <= np.finfo(float).eps: + msg = "All the data is zero in device dump data." + return CompareConst.NA, False, msg + elif b_value_max <= np.finfo(float).eps: + msg = "All the data is zero in bench dump data." + return CompareConst.NA, False, msg + else: + n_value = n_value.astype(float) / n_value_max + b_value = b_value.astype(float) / b_value_max + cos = np.dot(n_value, b_value) / (np.linalg.norm(n_value) * np.linalg.norm(b_value)) + if np.isnan(cos): + msg = "Dump data has NaN when comparing with Cosine Similarity." + cos = np.clip(cos, -1, 1) + return cos, cos > 0.99, msg + + +#rmse +def get_rmse(abs_err, inf_nan_mask): + masked_ae = np.where(inf_nan_mask, 0, abs_err) + mse = np.mean(np.square(masked_ae)) + inf_nan_cnt = np.sum(inf_nan_mask) + mse = mse * (abs_err.size / (abs_err.size - inf_nan_cnt + 0.0001) + 0.0001) + rmse = np.sqrt(mse) + return rmse + + +#误差均衡性 +def get_error_balance(bench_data, device_data): + larger_count = np.sum(np.greater(device_data - bench_data.astype(device_data.dtype), 0)) + smaller_count = np.sum(np.less(device_data - bench_data.astype(device_data.dtype), 0)) + total_count = bench_data.size + error_balance = abs(larger_count - smaller_count) / total_count if total_count > 0 else 0 + return error_balance + + +#小值域错误占比 +def get_small_value_err_ratio(small_value_mask, abs_err_greater_mask): + err_mask = np.logical_and(small_value_mask, abs_err_greater_mask) + small_value_err_num = np.sum(err_mask) + small_value_num = np.sum(small_value_mask) + return 0 if small_value_num == 0 else small_value_err_num / small_value_num + + +def get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask): + rel_err_tmp = abs_err / abs_bench_with_eps + rel_err_mask = np.logical_or(small_value_mask, inf_nan_mask) + rel_err = np.where(rel_err_mask, -1, rel_err_tmp) + return rel_err + + +def get_abs_err(bench_data, device_data): + abs_err = np.abs(device_data - bench_data) + return abs_err + + +def get_rel_err_origin(abs_err, b_value): + rel_err_origin = np.abs(abs_err / b_value) + return rel_err_origin + + +def get_max_abs_err(abs_err): + max_abs_err = abs_err.max() + bool_result = max_abs_err < 0.001 + return max_abs_err, bool_result + + +#相对误差最大值 +def get_max_rel_err(rel_err): + return np.max(rel_err) + + +#相对误差均值 +def get_mean_rel_err(rel_err): + return np.mean(rel_err) + + +def get_rel_err_ratio(rel_err, thresholding): + if np.size(rel_err) == 0: + ratio = 1 + else: + ratio = np.divide(np.sum(rel_err < thresholding), np.size(rel_err)) + bool_result = ratio > (1 - thresholding) + return ratio, bool_result + + +def get_finite_and_infinite_mask(bench_output, device_output): + device_finite_mask = np.isfinite(device_output) + bench_finite_mask = np.isfinite(bench_output.astype(device_output.dtype)) + both_finite_mask = np.logical_and(device_finite_mask, bench_finite_mask) + inf_nan_mask = np.logical_not(both_finite_mask) + return both_finite_mask, inf_nan_mask + + +def get_small_value_mask(abs_bench, both_finite_mask, small_value_threshold): + small_value_mask = np.less_equal(abs_bench, small_value_threshold) + small_value_mask = np.logical_and(small_value_mask, both_finite_mask) + return small_value_mask + + +def get_msg_and_handle_value(b_value, n_value): + if n_value.dtype in Const.FLOAT_TYPE: + zero_mask = (n_value == 0) + n_value[zero_mask] += np.finfo(n_value.dtype).eps + b_value[zero_mask] += np.finfo(n_value.dtype).eps + else: + b_value, n_value = b_value.astype(float), n_value.astype(float) + zero_mask = (n_value == 0) + n_value[zero_mask] += np.finfo(float).eps + b_value[zero_mask] += np.finfo(float).eps + return b_value, n_value + + +def compare_bool_tensor(bench_output, npu_out): + if npu_out.size == 0: + return CompareConst.NAN, CompareConst.ERROR, "There is not npu calculation result." + error_nums = (bench_output!= npu_out).sum() + error_rate = float(error_nums / bench_output.size) + result = CompareConst.PASS if error_rate == 0 else CompareConst.ERROR + return error_rate, result, "" + + +def compare_float_tensor(cpu_output, npu_output, compare_column): + npu_dtype = npu_output.dtype + + message = "" + eps = np.finfo(cpu_output.dtype).eps + abs_bench = np.abs(cpu_output) + abs_bench_with_eps = abs_bench + eps + abs_err = get_abs_err(cpu_output, npu_output) + if npu_dtype in [np.float16, np.float32]: + dtype_config = precision_configs.get(str(npu_dtype)) + both_finite_mask, inf_nan_mask = get_finite_and_infinite_mask(cpu_output, npu_output) + small_value_mask = get_small_value_mask(abs_bench, both_finite_mask, dtype_config['small_value'][0]) + abs_err_greater_mask = np.greater(abs_err, dtype_config['small_value_atol'][0]) + compare_column.small_value_err_ratio = get_small_value_err_ratio(small_value_mask, abs_err_greater_mask) + rel_err = get_rel_err(abs_err, abs_bench_with_eps, small_value_mask, inf_nan_mask) + compare_column.RMSE = get_rmse(abs_err, np.logical_or(inf_nan_mask, small_value_mask)) + compare_column.EB = get_error_balance(cpu_output, npu_output) + compare_column.Max_rel_error = get_max_rel_err(rel_err) + compare_column.Mean_rel_error = get_mean_rel_err(rel_err) + + + cos_res, cos_status, msg = cosine_sim(cpu_output, npu_output) + compare_column.cosine_sim = cos_res + message += msg + "\n" + if not cos_status: + return CompareConst.ERROR, compare_column, message + + max_abs_res, max_abs_status = get_max_abs_err(abs_err) + compare_column.max_abs_err = max_abs_res + if max_abs_status: + return CompareConst.PASS, compare_column, message + + + rel_err_orign = get_rel_err_origin(abs_err, abs_bench_with_eps) + if npu_dtype in [np.float16]: + hundred_res, hundred_status = get_rel_err_ratio(rel_err_orign, 0.01) + compare_column.rel_err_hundredth = hundred_res + if not hundred_status: + return CompareConst.ERROR, compare_column, message + thousand_res, thousand_status = get_rel_err_ratio(rel_err_orign, 0.001) + compare_column.rel_err_thousandth = thousand_res + if npu_dtype in [np.float16]: + if thousand_status: + return CompareConst.PASS, compare_column, message + return CompareConst.WARNING, compare_column, message + ten_thousand_res, ten_thousand_status = get_rel_err_ratio(rel_err_orign, 0.0001) + compare_column.rel_err_ten_thousandth = ten_thousand_res + if npu_dtype in [np.float32, np.float64]: + if not thousand_status: + return CompareConst.ERROR, compare_column, message + if not ten_thousand_status: + return CompareConst.WARNING, compare_column, message + return CompareConst.PASS, compare_column, message + + +class CompareColumn: + def __init__(self): + self.bench_type = CompareConst.NA + self.npu_type = CompareConst.NA + self.shape = CompareConst.NA + self.cosine_sim = CompareConst.NA + self.max_abs_err = CompareConst.NA + self.rel_err_hundredth = CompareConst.NA + self.rel_err_thousandth = CompareConst.NA + self.rel_err_ten_thousandth = CompareConst.NA + self.error_rate = CompareConst.NA + self.EB = CompareConst.NA + self.RMSE = CompareConst.NA + self.small_value_err_ratio = CompareConst.NA + self.Max_rel_error = CompareConst.NA + self.Mean_rel_error = CompareConst.NA + + def to_column_value(self, is_pass, message): + return [self.bench_type, self.npu_type, self.shape, self.cosine_sim, self.max_abs_err, self.rel_err_hundredth, + self.rel_err_thousandth, self.rel_err_ten_thousandth, self.error_rate, self.EB, self.RMSE, + self.small_value_err_ratio, self.Max_rel_error, self.Mean_rel_error, is_pass, message] + + + + diff --git a/debug/accuracy_tools/api_checker/compare/compare.py b/debug/accuracy_tools/api_checker/compare/compare.py new file mode 100644 index 0000000000000000000000000000000000000000..4688bb8d3476201ee0f0f65dd3d456509ee12b5b --- /dev/null +++ b/debug/accuracy_tools/api_checker/compare/compare.py @@ -0,0 +1,76 @@ +import os +from datetime import datetime, timezone +from compare.algorithm import CompareColumn, compare_float_tensor, compare_bool_tensor +from common.utils import get_json_contents, write_csv, np_scalar_type + + +class Comparator: + # consts for result csv + RESULT_CSV_PATH = "_result.csv" + DETAILS_CSV_PATH = "_detail.csv" + + def __init__(self, outpath, is_continue_run_ut, stack_info_json_path=None): + time = datetime.now(tz=timezone.utc).strftime("%Y%m%d%H%M%S") + self.save_path = os.path.join(outpath, time + self.RESULT_CSV_PATH) + self.detail_save_path = os.path.join(outpath, time + self.DETAILS_CSV_PATH) + if not is_continue_run_ut and not os.path.exists(self.save_path) and not os.path.exists(self.detail_save_path): + self.write_csv_title() + if stack_info_json_path: + self.stack_info = get_json_contents(stack_info_json_path) + else: + self.stack_info = None + + self.test_result_cnt = { + "forward_fail_num": 0, "backward_fail_num": 0, "forward_and_backward_fail_num": 0, "success_num": 0, + "total_num": 0, "forward_or_backward_fail_num": 0 + } + + def compare(self, bench_out, npu_out, api_name): + compareColumn = CompareColumn() + compareColumn.bench_type = bench_out.dtype + compareColumn.npu_type = npu_out.dtype + compareColumn.shape = npu_out.shape + if npu_out.dtype in np_scalar_type: + err_rate, status, message = compare_bool_tensor(bench_out, npu_out) + compareColumn.err_rate = err_rate + else: + status, compareColumn, message = compare_float_tensor(bench_out, npu_out, compareColumn) + result_list = [api_name, status] + write_csv(result_list, self.save_path) + detail_list = [api_name] + detail_temp = compareColumn.to_column_value(status, message) + for detail in detail_temp: + detail_list.append(detail) + write_csv(detail_list, self.detail_save_path) + + def write_result_csv(self, detail_dict): + result_list = [detail_dict["api_name"], detail_dict["status"]] + write_csv(result_list, self.save_path) + + def write_csv_title(self): + result_test_rows = [ + "API name", + "Forward Test Success", + "Backward Test Success", + "Message" + ] + write_csv(result_test_rows, self.save_path) + + detail_test_rows = [ + "API Name", "Bench Dtype", "NPU Dtype", "Shape", + "余弦相似度", + "最大绝对误差", + "双百指标", + "双千指标", + "双万指标", + "错误率", + "误差均衡性", + "均方根误差", + "小值域错误占比", + "相对误差最大值", + "相对误差平均值", + "Status", + "Message" + ] + write_csv(detail_test_rows, self.detail_save_path) + diff --git a/debug/accuracy_tools/api_checker/compare/compare_utils.py b/debug/accuracy_tools/api_checker/compare/compare_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6dea015d7bf3d2e663e455c985df569292566473 --- /dev/null +++ b/debug/accuracy_tools/api_checker/compare/compare_utils.py @@ -0,0 +1,50 @@ +from common.utils import Const, print_warn_log +import numpy as np + + +class CompareConst: + NAN = np.nan + NA = "N/A" + PASS = 'pass' + WARNING = 'warning' + ERROR = 'error' + SKIP = 'SKIP' + TRUE = 'TRUE' + FALSE = 'FALSE' + + +def check_dtype_comparable(x, y): + if x.dtype in Const.FLOAT_TYPE: + if y.dtype in Const.FLOAT_TYPE: + return True + return False + if x.dtype in Const.BOOL_TYPE: + if y.dtype in Const.BOOL_TYPE: + return True + return False + if x.dtype in Const.INT_TYPE: + if y.dtype in Const.INT_TYPE: + return True + return False + print_warn_log(f"Compare: Unexpected dtype {x.dtype}, {y.dtype}") + return False + + +precision_configs = { + 'float16' : { + 'small_value' : [ + 1e-3 + ], + 'small_value_atol' : [ + 1e-5 + ] + }, + 'float32':{ + 'small_value' : [ + 1e-6 + ], + 'small_value_atol' : [ + 1e-9 + ] + } +} diff --git a/debug/accuracy_tools/api_checker/main.py b/debug/accuracy_tools/api_checker/main.py new file mode 100644 index 0000000000000000000000000000000000000000..86d2f4a5959c24f7b997494c02225c53ded4b0b7 --- /dev/null +++ b/debug/accuracy_tools/api_checker/main.py @@ -0,0 +1,50 @@ +import argparse +import json +import os +from common.logger import logger + + +def main(): + parser = argparse.ArgumentParser(description='Generate JSON based on user input') + + parser.add_argument('--mode', type=int, default=0, help='Dump mode (0 for all data and 1 for select api)') + parser.add_argument('--out', type=str, default="./data", help='Dump data output dir') + parser.add_argument('--net_name', type=str, default="MyNet", help='Network name (e.g., MyNet)') + parser.add_argument('--iteration', type=str, default="0", help='Iteration range (e.g., "0|5-8|100-120")') + parser.add_argument('--saved_data', type=str, default="tensor", help='Saved data type ("tensor" for dump tensor and "statistic" for statistic data)') + parser.add_argument('--input_output', type=int, default="0", help='Input output flag (0 for all and 1 for input and 2 for output)') + parser.add_argument('--kernels', type=str, nargs='+', default="", help='List of selected kernels only valid when mode is 1(e.g., "Default/Conv-op12")') + parser.add_argument('--support_device', type=int, nargs='+', required=True, help='List of supported devices (e.g., 0 1 2 3 4 5 6 7)') + parser.add_argument('--e2e_enable', type=bool, default=True, help='Enable end-to-end dump (true/false)') + parser.add_argument('--e2e_trans_flag', type=bool, default=True, help='End-to-end trans flag (true/false)') + parser.add_argument('--output_dir', type=str, default="./", help='Output JSON file path') + + args = parser.parse_args() + + json_data = { + "common_dump_settings": { + "dump_mode": args.mode, + "path": os.path.realpath(args.out), + "net_name": args.net_name, + "iteration": args.iteration, + "saved_data": args.saved_data, + "input_output": args.input_output, + "kernels": list(args.kernels), + "support_device": list(args.support_device), + "op_debug_mode": 0, + "file_format": "npy" + }, + "e2e_dump_settings": { + "enable": args.e2e_enable, + "trans_flag": args.e2e_trans_flag, + "save_kernel_args": True + } + } + dump_json_path = os.path.realpath(args.output_dir) + output_json_path = os.path.join(args.output_dir, "dump.json") + with open(output_json_path, 'w') as f: + json.dump(json_data, f, indent=4) + + logger.info(f"JSON data saved to {output_json_path}") +if __name__ == '__main__': + main() diff --git a/debug/accuracy_tools/api_checker/run_ut.py b/debug/accuracy_tools/api_checker/run_ut.py new file mode 100644 index 0000000000000000000000000000000000000000..2c3bf680cfac22910c6e3dfba498db66abc86296 --- /dev/null +++ b/debug/accuracy_tools/api_checker/run_ut.py @@ -0,0 +1,122 @@ +import os +import sys +import json +import importlib +import inspect +import argparse +from collections import defaultdict +import numpy as np +import pandas as pd +from compare.compare import Comparator +from common.logger import logger + + +def _run_ut_parser(parser): + parser.add_argument( + "-i", "--input", dest="input_file", required=True, type=str, + help=" Input josn file containing the API information" + ) + parser.add_argument( + "-o", "--output", dest="output_path", required=False, type=str, + help=" Output path to store the comparison result" + ) + + +def get_ops_ut(module): + for name, obj in inspect.getmembers(module): + if inspect.isclass(obj) and name.startswith("UT"): + return obj + + +def load_json(file_path): + with open(file_path, 'r') as f: + return json.load(f) + + +def load_npy(file_path): + return np.load(file_path) + + +def _run_ut(): + parser = argparse.ArgumentParser() + _run_ut_parser(parser) + args = parser.parse_args(sys.argv[1:]) + out_path = os.path.realpath(args.output_path) if args.output_path else "./" + data_path = os.path.realpath(args.input_file) + comparator = Comparator(out_path, False) + cur_path = os.path.dirname(os.path.realpath(__file__)) + api_mapping_path = os.path.join(cur_path, "api_mapping.json") + + with open(api_mapping_path, 'r') as f: + api_mapping_dict = json.load(f) + + + file_groups = defaultdict(lambda: {'json': None, 'args': [], 'output': []}) + + for dirpath, _, filenames in os.walk(data_path): + for filename in filenames: + file_path = os.path.join(dirpath, filename) + base_name, ext = os.path.splitext(filename) + if ext == ".csv": + continue + real_name = base_name + if real_name.isdigit(): + mapping_csv_path = os.path.join(dirpath, "mapping.csv") + mapping_df = pd.read_csv(mapping_csv_path, header=None, names=['filename', 'realname']) + mapping_col = mapping_df[mapping_df['filename'] == filename] + real_name = mapping_col['realname'].tolist()[0] + + parts = real_name.split('.') + op_type = parts[0] + op_name = parts[1] + file_groups[op_name]['type'] = op_type + if ext == ".json": + file_groups[op_name]['json'] = file_path + elif ext == ".npy": + if 'input' in real_name: + file_groups[op_name]['args'].append(file_path) + elif 'output' in real_name: + file_groups[op_name]['output'].append(file_path) + + for op_name, files in file_groups.items(): + json_path = files['json'] + input_paths = files['args'] + output_paths = files['output'] + op_type = files['type'] + if op_type in api_mapping_dict: + module_name = "common_ut" + api_name = op_type + "_" + api_mapping_dict[op_type] + else: + module_name = op_type + "_ut" + api_name = op_name + + if not os.path.exists(f"ut_cast/{module_name}.py"): + logger.warning(f"{op_type} not support compare now") + continue + + if json_path: + kwargs = load_json(json_path) + args = [load_npy(npy_path) for npy_path in sorted(input_paths)] + output = [load_npy(npy_path) for npy_path in sorted(output_paths)] + module = importlib.import_module(f"ut_case.{module_name}") + if not module: + logger.warning(f"load {module} failed") + continue + + ops_ut = get_ops_ut(module) + try: + ops_ut( + api_name, + args, + kwargs, + output, + real_data=True, + stack=None, + comparator=comparator + ).compare() + except Exception as e: + logger.warning(f">>>[{op_name}] Compare failed.Reason: {e}") + + +if __name__ == '__main__': + _run_ut() \ No newline at end of file diff --git a/debug/accuracy_tools/api_checker/ut_base.py b/debug/accuracy_tools/api_checker/ut_base.py new file mode 100644 index 0000000000000000000000000000000000000000..e55d79d08b94b6e013d9b2d2116ea88e660024f0 --- /dev/null +++ b/debug/accuracy_tools/api_checker/ut_base.py @@ -0,0 +1,79 @@ +import numpy as np +import os +import mindspore as ms +import torch +from common.utils import Const, dtype_map +from collections import deque +from collections import defaultdict + + +class UTBase: + def __init__(self, name, args, kwargs, output=None, real_data=False, stack=None, comparator=None): + self.name = name + self.args = args + self.kwargs = kwargs + self.output = output + self.real_data = real_data + self.stack = stack + self.comparator = comparator + + @staticmethod + def convert_list_to_tuple(data): + for key, value in data.items(): + if not isinstance(value, list): + continue + data[key] = tuple(value) + return data + + def insert_into_dict(self, dic, keys, value): + key = keys.pop(0) + if not keys: + if key in dic: + dic[key].append(value) + else: + dic[key] = [value] + else: + if key not in dic: + dic[key] = defaultdict(list) + self.insert_into_dict(dic[key], keys, value) + + def forward_mindspore_impl(self, *input): + pass + + def forward_pytorch_impl(self, *input): + pass + + def forward_cmp_real(self): + data_input = self.args + input_pt = [] + for data in data_input: + if data.shape: + input_pt.append(torch.from_numpy(data)) + else: + origin_data = data.item() + if data.dtype == np.int64: + origin_data = int(origin_data) + elif data.dtype == np.float64: + origin_data = float(origin_data) + input_pt.append(origin_data) + + output_pt = self.forward_pytorch_impl(*input_pt) + + if isinstance(output_pt, torch.Tensor): + output_ms = self.output[0] + output_pt = output_pt.numpy() + self.comparator.compare(output_pt, output_ms, self.name + "." + Const.INPUT) + else: + for index_output, (output_p, output_ms) in enumerate(zip(output_pt, self.output)): + output_p = output_p.numpy() + output_ms = np.load(os.path.join(self.save_path, output_ms)) + self.comparator.compare(output_p, output_ms, self.name + "." + Const.OUTPUT + "." + str(index_output)) + + def forward_cmp_random(self): + pass + + def compare(self): + if self.real_data: + self.forward_cmp_real() + else: + self.forward_cmp_random() \ No newline at end of file diff --git a/debug/accuracy_tools/api_checker/ut_case/Add_ut.py b/debug/accuracy_tools/api_checker/ut_case/Add_ut.py new file mode 100644 index 0000000000000000000000000000000000000000..e462575ff3c16312f8275d582055103273f14b47 --- /dev/null +++ b/debug/accuracy_tools/api_checker/ut_case/Add_ut.py @@ -0,0 +1,37 @@ +import mindspore as ms +from mindspore.nn.cell import Cell +from mindspore.ops import operations as P +import torch +from ut_base import UTBase +from common.logger import logger + + +class Add(Cell): + def __init__(self): + super().__init__() + self.add = P.Add() + + def construct(self, input_x, input_y): + return self.add(input_x, input_y) + + +class AddUT(UTBase): + def __init__(self, name, args, kwargs, output, real_data=False, stack=None, comparator=None): + super().__init__(name, args, kwargs, output, real_data, stack, comparator) + + def forward_mindspore_impl(self, *args): + x = args[0] + y = args[1] + net = Add() + out = net(x, y) + return out + + def forward_pytorch_impl(self, *args): + input_pt_x = args[0] + input_pt_y = args[1] + if not isinstance(input_pt_x, torch.Tensor): + input_pt_x = torch.tensor(input_pt_x) + output = torch.add(input_py_x, input_pt_y) + if output.dtype == torch.bfloat16: + return output.float() + return output \ No newline at end of file diff --git a/debug/accuracy_tools/api_checker/ut_case/ArgMaxWithValue_ut.py b/debug/accuracy_tools/api_checker/ut_case/ArgMaxWithValue_ut.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/debug/accuracy_tools/api_checker/ut_case/MatMul_ut.py b/debug/accuracy_tools/api_checker/ut_case/MatMul_ut.py new file mode 100644 index 0000000000000000000000000000000000000000..40373334b2dc8b1a31f8a22e978fcd6dd324e312 --- /dev/null +++ b/debug/accuracy_tools/api_checker/ut_case/MatMul_ut.py @@ -0,0 +1,44 @@ +import mindspore as ms +from mindspore.nn.cell import Cell +from mindspore.ops import operations as P +from mindspore.common import dtype as mstype +import torch +from ut_base import UTBase + + +class MatMul(Cell): + def __init__(self, transponse_a=False, transponse_b=False): + super().__init__() + self.matmul = P.MatMul(transponse_a, transponse_b) + + def construct(self, input_x, input_y): + return self.matmul(input_x, input_y) + + +class MatMulUT(UTBase): + def __init__(self, name, args, kwargs, output, real_data=False, stack=None, comparator=None): + super().__init__(name, args, kwargs, output, real_data, stack, comparator) + len_args = len(args) + self.transpose_a = self.kwargs.get("transpose_a") if self.kwargs else False + self.transpose_b = self.kwargs.get("transpose_b") if self.kwargs else False + + def forward_mindspore_impl(self, *args): + x = args[0] + y = args[1] + net = MatMul(self.transpose_a, self.transpose_a) + out = net(x, y) + if out.dtype == mstype.bfloat16: + return out.float().asnumpy() + return out + + def forward_pytorch_impl(self, *args): + input_pt_x = args[0] + input_pt_y = args[1] + if self.transpose_a: + input_pt_x = torch.transpose(input_pt_x, 0, 1) + if self.transpose_b: + input_pt_y = torch.transpose(input_pt_y, 0, 1) + output = torch.matmul(input_pt_x, input_pt_y) + if output.dtype == torch.bfloat16: + return output.float().numpy() + return output \ No newline at end of file diff --git a/debug/accuracy_tools/api_checker/ut_case/common_ut.py b/debug/accuracy_tools/api_checker/ut_case/common_ut.py new file mode 100644 index 0000000000000000000000000000000000000000..7338d20c900e4a2f1a1866f8ade14cafddffe424 --- /dev/null +++ b/debug/accuracy_tools/api_checker/ut_case/common_ut.py @@ -0,0 +1,39 @@ +import mindspore as ms +from mindspore.nn.cell import Cell +from mindspore.ops import operations as P +import torch +import re +import inspect +from ut_base import UTBase +from common.logger import logger + + +class Common(Cell): + def __init__(self): + super().__init__() + + +class CommonUT(UTBase): + def __init__(self, name, args, kwargs, output, real_data=False, stack=None, comparator=None): + super().__init__(name, args, kwargs, output, real_data, stack, comparator) + + pattern = re.compile(r"^(.*?)_(.*)$") + match = pattern.match(self.name) + + if match: + self.name_ms = match.group(1) + logger.info(f"Common UT compare mindspore api: {self.name_ms}") + self.name_py = match.group(2) + self.name = self.name_ms + "_" + self.name + else: + logger.warning("No match UT found") + + def forward_mindspore_impl(self, *args): + output_ms = getattr(P, self.name_ms)()(*args) + return output_ms + + def forward_pytorch_impl(self, *args): + args_len = len(inspect.getfullargspec(getattr(P, self.name_ms)()).args) - 1 + tensor_list = args[:args_len] + output_py = getattr(torch,self.name_py)(*tensor_list) + return output_py \ No newline at end of file