diff --git a/debug/accuracy_tools/api_accuracy_checker/README.md b/debug/accuracy_tools/api_accuracy_checker/README.md index 5ff4b26868afa0319dcb9ef723b5081411bb372a..1260a58334968865e0cf38e3a66a6f3c45426344 100644 --- a/debug/accuracy_tools/api_accuracy_checker/README.md +++ b/debug/accuracy_tools/api_accuracy_checker/README.md @@ -51,7 +51,7 @@ Ascend模型精度预检工具能在昇腾NPU上扫描用户训练模型中所 forward和backward两个命令行参数根据实际存盘的json文件名配置。比对结果存盘路径默认是'./'(相对于run_ut的路径),可以在运行run_ut.py时通过 --out_path命令行参数配置。结果包括pretest_result.csv和pretest_details.csv两个文件。前者是api粒度的,标明每个api是否通过测试。建议用户先查看前者,对于其中没有通过测试的或者特定感兴趣的api,根据其API name字段在pretest_details.csv中查询其各个输出的达标情况以及比较指标。 - 注意:目前API通过测试的标准是每个输出与标杆比对的余弦相似度大于0.99,pretest_details.csv中的相对误差供用户分析时使用。 + 注意:目前API通过测试的标准是每个输出与标杆比对的余弦相似度大于0.99,并且float16数据要通过双千分之一标准,float32数据要通过双万分之一标准,pretest_details.csv中的相对误差供用户分析时使用。 ## FAQ diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index c8ecea17b01d62de6d3ceef7c47ce339861a87e3..29b10608e496a7808a13dcb8656521e6bcc8cba8 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -70,7 +70,7 @@ def get_rel_err_ratio_ten_thousandth(n_value, b_value): ratio, bool_result, msg = get_rel_err_ratio(n_value, b_value, 0.0001) if b_value.dtype == np.float16: msg = f"This indicator is not used to evaluate {b_value.dtype} data" - return ratio, CompareConst.NA, msg + return ratio, True, msg return ratio, bool_result, msg def get_rel_err_ratio(n_value, b_value, thresholding): @@ -187,7 +187,7 @@ def compare_core(bench_out, npu_out, alg): if isinstance(compare_result, list): compare_result = flatten_compare_result(compare_result) else: - compare_result = [(compare_result, str(test_success), msg)] + compare_result = [(compare_result, msg)] if isinstance(bench_dtype, list): bench_dtype = flatten_compare_result(bench_dtype) npu_dtype = flatten_compare_result(npu_dtype) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index dd2d5f3b0f56c7d169897e61f2024b879ad919ba..c5ae3729947ed75eaefe439b3d9a2fd6c243ef1b 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -68,12 +68,12 @@ class Comparator: def write_detail_csv(self): test_rows = [[ "Subject", "Bench Dtype", "NPU Dtype", - "Cosine Similarity", "Cosine Similarity Pass", "Cosine Similarity Message", - "Max Rel Error", "Max Rel Err Pass", "Max Rel Err Message", - "Thousandth Rel Error Ratio", "Thousandth Rel Error Ratio Pass", "Thousandth Rel Error Ratio Message", - "Ten Thousandth Rel Error Ratio", "Ten Thousandth Rel Error Ratio Pass", "Ten Thousandth Rel Error Ratio Message", - "Compare Builtin Type", "Builtin Type Pass", - "Builtin Type Message" + "Cosine Similarity", "Cosine Similarity Message", + "Max Rel Error", "Max Rel Err Message", + "Thousandth Rel Error Ratio", "Thousandth Rel Error Ratio Message", + "Ten Thousandth Rel Error Ratio", "Ten Thousandth Rel Error Ratio Message", + "Compare Builtin Type", "Builtin Type Message", + "Pass" ]] for test_result in self.test_results: subject_prefix = test_result[0] @@ -129,18 +129,19 @@ class Comparator: detailed_result, test_success, bench_dtype, npu_dtype = compare_core(bench_out, npu_out, alg) bench_dtype_total = bench_dtype npu_dtype_total = npu_dtype - if name != "Max Relative Error" and test_success != CompareConst.NA: + if name != "Max Relative Error": test_success_total = test_success_total and test_success if detailed_result_total: for i in range(len(detailed_result_total)): detailed_result_total[i] += detailed_result[i] else: detailed_result_total = detailed_result - # dtype加到所有指标的前面 + # dtype加到所有指标的前面, 是否pass放到所有指标的后面 for i in range(len(detailed_result_total)): detailed_result = list(detailed_result_total[i]) detailed_result.insert(0, bench_dtype_total[i]) detailed_result.insert(1, npu_dtype_total[i]) + detailed_result.append(str(test_success_total)) detailed_result_total[i] = tuple(detailed_result) return test_success_total, detailed_result_total