From 69ebc24fd15dfb8aec3c9ce6e9b346f827ce7350 Mon Sep 17 00:00:00 2001 From: l30044004 Date: Mon, 21 Aug 2023 20:08:25 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E6=AF=8F=E4=B8=AA?= =?UTF-8?q?=E6=8C=87=E6=A0=87=E5=90=8E=E7=9A=84True/False=EF=BC=8C?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=B8=80=E4=B8=AAPass=E5=88=97=E4=BB=A3?= =?UTF-8?q?=E8=A1=A8=E6=AF=94=E5=AF=B9=E7=BB=93=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api_accuracy_checker/compare/algorithm.py | 2 +- .../api_accuracy_checker/compare/compare.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index c8ecea17b0..595c189f50 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -187,7 +187,7 @@ def compare_core(bench_out, npu_out, alg): if isinstance(compare_result, list): compare_result = flatten_compare_result(compare_result) else: - compare_result = [(compare_result, str(test_success), msg)] + compare_result = [(compare_result, msg)] if isinstance(bench_dtype, list): bench_dtype = flatten_compare_result(bench_dtype) npu_dtype = flatten_compare_result(npu_dtype) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index dd2d5f3b0f..0c4005912a 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -68,12 +68,12 @@ class Comparator: def write_detail_csv(self): test_rows = [[ "Subject", "Bench Dtype", "NPU Dtype", - "Cosine Similarity", "Cosine Similarity Pass", "Cosine Similarity Message", - "Max Rel Error", "Max Rel Err Pass", "Max Rel Err Message", - "Thousandth Rel Error Ratio", "Thousandth Rel Error Ratio Pass", "Thousandth Rel Error Ratio Message", - "Ten Thousandth Rel Error Ratio", "Ten Thousandth Rel Error Ratio Pass", "Ten Thousandth Rel Error Ratio Message", - "Compare Builtin Type", "Builtin Type Pass", - "Builtin Type Message" + "Cosine Similarity", "Cosine Similarity Message", + "Max Rel Error", "Max Rel Err Message", + "Thousandth Rel Error Ratio", "Thousandth Rel Error Ratio Message", + "Ten Thousandth Rel Error Ratio", "Ten Thousandth Rel Error Ratio Message", + "Compare Builtin Type", "Builtin Type Message", + "Pass" ]] for test_result in self.test_results: subject_prefix = test_result[0] @@ -129,18 +129,19 @@ class Comparator: detailed_result, test_success, bench_dtype, npu_dtype = compare_core(bench_out, npu_out, alg) bench_dtype_total = bench_dtype npu_dtype_total = npu_dtype - if name != "Max Relative Error" and test_success != CompareConst.NA: + if name not in ["Cosine Similarity", "Max Relative Error"] and test_success != CompareConst.NA: test_success_total = test_success_total and test_success if detailed_result_total: for i in range(len(detailed_result_total)): detailed_result_total[i] += detailed_result[i] else: detailed_result_total = detailed_result - # dtype加到所有指标的前面 + # dtype加到所有指标的前面, 是否pass放到所有指标的后面 for i in range(len(detailed_result_total)): detailed_result = list(detailed_result_total[i]) detailed_result.insert(0, bench_dtype_total[i]) detailed_result.insert(1, npu_dtype_total[i]) + detailed_result.append(test_success_total) detailed_result_total[i] = tuple(detailed_result) return test_success_total, detailed_result_total -- Gitee From 34f3c18efed6e00ac60ecd4447af21cbd2799251 Mon Sep 17 00:00:00 2001 From: l30044004 Date: Tue, 22 Aug 2023 09:54:10 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E6=AF=8F=E4=B8=AA?= =?UTF-8?q?=E6=8C=87=E6=A0=87=E5=90=8E=E7=9A=84True/False=EF=BC=8C?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=B8=80=E4=B8=AAPass=E5=88=97=E4=BB=A3?= =?UTF-8?q?=E8=A1=A8=E6=AF=94=E5=AF=B9=E7=BB=93=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/api_accuracy_checker/compare/compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 0c4005912a..b8d425c5b6 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -141,7 +141,7 @@ class Comparator: detailed_result = list(detailed_result_total[i]) detailed_result.insert(0, bench_dtype_total[i]) detailed_result.insert(1, npu_dtype_total[i]) - detailed_result.append(test_success_total) + detailed_result.append(str(test_success_total)) detailed_result_total[i] = tuple(detailed_result) return test_success_total, detailed_result_total -- Gitee From 98aaf78fb4b76b0c867d272405cc0dd3541930e8 Mon Sep 17 00:00:00 2001 From: l30044004 Date: Tue, 22 Aug 2023 14:10:37 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E6=AF=8F=E4=B8=AA?= =?UTF-8?q?=E6=8C=87=E6=A0=87=E5=90=8E=E7=9A=84True/False=EF=BC=8C?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=B8=80=E4=B8=AAPass=E5=88=97=E4=BB=A3?= =?UTF-8?q?=E8=A1=A8=E6=AF=94=E5=AF=B9=E7=BB=93=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/api_accuracy_checker/README.md | 2 +- debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py | 2 +- debug/accuracy_tools/api_accuracy_checker/compare/compare.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/README.md b/debug/accuracy_tools/api_accuracy_checker/README.md index 5ff4b26868..1260a58334 100644 --- a/debug/accuracy_tools/api_accuracy_checker/README.md +++ b/debug/accuracy_tools/api_accuracy_checker/README.md @@ -51,7 +51,7 @@ Ascend模型精度预检工具能在昇腾NPU上扫描用户训练模型中所 forward和backward两个命令行参数根据实际存盘的json文件名配置。比对结果存盘路径默认是'./'（相对于run_ut的路径），可以在运行run_ut.py时通过 --out_path命令行参数配置。结果包括pretest_result.csv和pretest_details.csv两个文件。前者是api粒度的，标明每个api是否通过测试。建议用户先查看前者，对于其中没有通过测试的或者特定感兴趣的api，根据其API name字段在pretest_details.csv中查询其各个输出的达标情况以及比较指标。 - 注意：目前API通过测试的标准是每个输出与标杆比对的余弦相似度大于0.99，pretest_details.csv中的相对误差供用户分析时使用。 + 注意：目前API通过测试的标准是每个输出与标杆比对的余弦相似度大于0.99，并且float16数据要通过双千分之一标准，float32数据要通过双万分之一标准，pretest_details.csv中的相对误差供用户分析时使用。 ## FAQ diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index 595c189f50..29b10608e4 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -70,7 +70,7 @@ def get_rel_err_ratio_ten_thousandth(n_value, b_value): ratio, bool_result, msg = get_rel_err_ratio(n_value, b_value, 0.0001) if b_value.dtype == np.float16: msg = f"This indicator is not used to evaluate {b_value.dtype} data" - return ratio, CompareConst.NA, msg + return ratio, True, msg return ratio, bool_result, msg def get_rel_err_ratio(n_value, b_value, thresholding): diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index b8d425c5b6..c5ae372994 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -129,7 +129,7 @@ class Comparator: detailed_result, test_success, bench_dtype, npu_dtype = compare_core(bench_out, npu_out, alg) bench_dtype_total = bench_dtype npu_dtype_total = npu_dtype - if name not in ["Cosine Similarity", "Max Relative Error"] and test_success != CompareConst.NA: + if name != "Max Relative Error": test_success_total = test_success_total and test_success if detailed_result_total: for i in range(len(detailed_result_total)): -- Gitee