diff --git a/debug/accuracy_tools/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/api_accuracy_checker/common/utils.py index 24959a62950f03974aaf2822dc5cc83747475cfd..daf32a2a19079bb981d6ba9dadfb97c63cc2889c 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/utils.py @@ -26,6 +26,7 @@ from datetime import datetime, timezone import numpy as np import torch +import pandas as pd try: import torch_npu @@ -60,6 +61,7 @@ class Const: OFF = 'OFF' BACKWARD = 'backward' FORWARD = 'forward' + FLOAT_TYPE = [np.half, np.single, np.double, np.float64, np.longdouble] # dump mode ALL = "all" @@ -75,7 +77,6 @@ class Const: WRITE_FLAGS = os.O_WRONLY | os.O_CREAT WRITE_MODES = stat.S_IWUSR | stat.S_IRUSR - class CompareConst: """ Class for compare module const @@ -167,6 +168,15 @@ class CompareException(Exception): class DumpException(CompareException): pass +def read_json(file): + with open(file, 'r') as f: + obj = json.load(f) + return obj + +def write_csv(data, filepath): + data_frame = pd.DataFrame(columns=data) + data_frame.to_csv(filepath, index=False) + def _print_log(level, msg): current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))) pid = os.getgid() diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/__init__.py b/debug/accuracy_tools/api_accuracy_checker/compare/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index cbb64cd196108bc45daa03f434d517c5f9486204..9ccdb05baa877c42488ca1ae4a5aa2bb0f66dbbc 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -1 +1,111 @@ -# 定义比对算法及比对标准 \ No newline at end of file +# 定义比对算法及比对标准 + +import torch +import numpy as np +from .compare_utils import CompareConst +from ..common.utils import print_warn_log, Const + +def compare_torch_tensor(cpu_output, npu_output, compare_alg): + if cpu_output.dtype == torch.bool: + return compare_bool_tensor(cpu_output, npu_output) + return compare_alg(cpu_output, npu_output) + + +def compare_bool_tensor(cpu_output, npu_output, compare_alg): + error_rate = CompareConst.NAN + cpu_shape = cpu_output.shape + npu_shape = npu_output.shape + if cpu_shape != npu_shape: + return error_rate, False + npu_data = npu_output.cpu().detach().numpy() + bench_data = cpu_output.detach().numpy() + data_size = bench_data.size + error_nums = (bench_data != npu_data).sum() + error_rate = float(error_nums / data_size) + return error_rate, error_rate < 0.001 + + +def get_max_rel_err(n_value, b_value): + if not isinstance(n_value, np.ndarray) or not isinstance(b_value, np.ndarray): + print_warn_log("Max rel err only support numpy array!") + raise ValueError("Max rel err only support numpy array!") + if n_value.dtype != b_value.dtype: + raise ValueError("npu and bench value dtype is different.") + if n_value.dtype in Const.FLOAT_TYPE: + rel_err = np.abs((n_value - b_value) / (b_value + np.finfo(b_value.dtype).eps)).max() + return rel_err, rel_err < 0.001 + if np.all(n_value == b_value): + return 0, True + return 1, False + + +def cosine_sim(cpu_output, npu_output): + n_value = npu_output.cpu().detach().numpy().reshape(-1) + b_value = cpu_output.detach().numpy().reshape(-1) + cos = CompareConst.NA + np.seterr(divide="ignore", invalid="ignore") + if len(n_value) == 1: + print_warn_log("All the data in npu dump data is scalar. Compare by relative error.") + return get_max_rel_err(n_value, b_value) + num = n_value.dot(b_value) + a_norm = np.linalg.norm(n_value) + b_norm = np.linalg.norm(b_value) + if a_norm <= np.finfo(float).eps and b_norm <= np.finfo(float).eps: + return cos, True + elif a_norm <= np.finfo(float).eps: + print_warn_log("All the data is Zero in npu dump data. Compare by relative error.") + return get_max_rel_err(n_value, b_value) + elif b_norm <= np.finfo(float).eps: + print_warn_log("All the data is Zero in bench dump data. Compare by relative error.") + else: + cos = num / (a_norm * b_norm) + if np.isnan(cos): + print_warn_log("Dump data has NaN when comparing with Cosine Similarity.") + return cos, cos > 0.99 + + +def compare_builtin_type(bench_out, npu_out): + if bench_out != npu_out: + return CompareConst.NAN, False + return 1.0, True + + +def flatten_compare_result(result): + flatten_result = [] + for result_i in result: + if isinstance(result_i, list): + flatten_result += flatten_compare_result(result_i) + else: + flatten_result.append(result_i) + return flatten_result + + +def compare_core(bench_out, npu_out, alg): + if type(bench_out) != type(npu_out): + raise ValueError("bench and npu output type is different") + if isinstance(bench_out, list, tuple): + compare_result, test_success = [], True + if len(bench_out) != len(npu_out): + raise ValueError("bench and npu output structure is different") + for b_out_i, n_out_i in zip(bench_out, npu_out): + compare_result_i, test_success_i = compare_core(b_out_i, n_out_i, alg) + compare_result.append(compare_result_i) + test_success = test_success and test_success_i + elif isinstance(bench_out, dict): + b_keys, n_keys = set(bench_out.keys()), set(npu_out.keys()) + if b_keys != n_keys: + raise ValueError("bench and npu output dictionary keys are different") + compare_result, test_success = compare_core(list(bench_out.values()), list(npu_out.values())) + elif isinstance(bench_out, torch.Tensor): + compare_result, test_success = compare_torch_tensor(bench_out, npu_out, alg) + elif isinstance(bench_out, (bool, int, float, str)): + compare_result, test_success = compare_builtin_type(bench_out, npu_out) + elif bench_out is None: + return 1.0, True + else: + raise NotImplementedError("Unexpected output type in compare_core: {}".format(type(bench_out))) + if isinstance(compare_result, list): + compare_result = flatten_compare_result(compare_result) + return compare_result, test_success + + diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index c4c64321701cd4b84967c267b4ed1bae68ba3447..eb1b2586e9a8625da4eac6a7995bca117a8e9b6c 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -1 +1,83 @@ -# 进行比对及结果展示 \ No newline at end of file +# 进行比对及结果展示 +import os +from prettytable import Prettytable +from .algorithm import compare_core, cosine_sim, cosine_standard +from ..common.utils import get_json_contents, print_error_log, print_info_log, write_csv +from .compare_utils import CompareConst + +class Comparator: + TEST_FILE_NAME = "pretest_result.csv" + # consts for result csv + COLUMN_API_NAME = "API name" + COLUMN_FORWARD_SUCCESS = "Forward Test Success" + COLUMN_BACKWARD_SUCCESS = "Backward Test Success" + COLUMN_STACK_INFO = "Traceback callstack info" + + def __init__(self, result_save_path, stack_info_json_path=None): + self.save_path = os.path.join(result_save_path, self.TEST_FILE_NAME) + if stack_info_json_path: + self.stack_info = get_json_contents(stack_info_json_path) + else: + self.stack_info = None + self.compare_alg = {} + self.compare_alg_names = [] + self.register_compare_algorithm("Cosine Similarity", cosine_sim, cosine_standard) + self.test_results = [] + self.test_result_cnt = {"forward_fail_num":0, "backward_fail_num":0, "forward_and_backward_fail_num":0, "success_num":0} + + def print_pretest_result(self): + res_dict = { + "forward_not_pass": self.test_result_cnt['forward_fail_num'], + "backward_not_pass": self.test_result_cnt['backward_fail_num'], + "forward_and_backward_not_pass": self.test_result_cnt['forward_and_backward_fail_num'], + "pass": self.test_result_cnt['success_num'] + } + tb = Prettytable() + tb.add_column("Category", list(res_dict.keys())) + tb.add_column("statistics",list(res_dict.values())) + info_tb = str(tb) + print_info_log(info_tb) + + def write_compare_csv(self): + self.write_summary_csv() + + def write_summary_csv(self): + test_rows = [[self.COLUMN_API_NAME, self.COLUMN_FORWARD_SUCCESS, self.COLUMN_BACKWARD_SUCCESS]] + if self.stack_info: + test_rows[0].append(self.COLUMN_STACK_INFO) + for result in self.test_results: + name = result[0] + df_row = list(result[:3]) + if self.stack_info: + stack_info = "\n".join(self.stack_info[name]) + df_row.append(stack_info) + test_rows.append(df_row) + write_csv(test_rows, self.save_path) + + def record_results(self, *args): + self.test_results.append(args) + + def register_compare_algorithm(self, name, compare_func, standard): + self.compare_alg.update({name: (compare_func, standard)}) + self.compare_alg_names.append(name) + + def compare_output(self, api_name, bench_out, npu_out, bench_grad=None, npu_grad=None): + is_fwd_success, fwd_compare_alg_results = self._compare_core_wrapper(bench_out, npu_out) + if bench_grad and npu_grad: + is_bwd_success, bwd_compare_alg_results = self._compare_core_wrapper(bench_grad, npu_grad) + else: + is_bwd_success, bwd_compare_alg_results = CompareConst.NA, None + self.record_results(api_name, is_fwd_success, is_bwd_success, fwd_compare_alg_results, bwd_compare_alg_results) + if is_fwd_success and is_bwd_success: + self.test_result_cnt['success_num'] += 1 + elif not is_fwd_success and not is_bwd_success: + self.test_result_cnt['forward_and_backward_fail_num'] += 1 + elif not is_fwd_success: + self.test_result_cnt['forward_fail_num'] += 1 + else: + self.test_result_cnt['backward_fail_num'] += 1 + + def _compare_core_wrapper(self, bench_out, npu_out): + name = self.compare_alg_names[0] + detailed_result, test_success = compare_core(bench_out, npu_out, self.compare_lag[name][0]) + return test_success, detailed_result \ No newline at end of file diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2f4d7eb38ccd5389b5d88a3bcdb6340ee6459ac7 --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -0,0 +1,4 @@ +import numpy as np +class CompareConst: + NAN = np.nan + NA = "N/A"