From ca5ddce9c52871702ae30bdbffe1e49cdf177845 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Thu, 14 Mar 2024 03:03:54 +0000 Subject: [PATCH 01/17] update debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml. Signed-off-by: jiangchangting1 --- .../compare/api_precision_standard.yaml | 534 +++--------------- 1 file changed, 91 insertions(+), 443 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml index 29665e9155..ceccf65a46 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_standard.yaml @@ -14,446 +14,94 @@ # limitations under the License. AbsoluteThreshStandard: - mul: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - mul_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __mul__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __imul__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __rmul__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - add: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - add_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __add__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __iadd__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __radd__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - div: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - div_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __div__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __idiv__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - divide: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - divide_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - floor_divide: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - floor_divide_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - true_divide: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - true_divide_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - leaky_relu: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - leaky_relu_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - prelu: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - reciprocal: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - reciprocal_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - rsqrt: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - rsqrt_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - square: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - square_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - sub: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - sub_: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - rsub: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __isub__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - __sub__: - torch.float32: - rtol: 0.000001 - small_value: 0.000001 - small_value_atol: 0.000001 - torch.float16: - rtol: 0.001 - small_value: 0.001 - small_value_atol: 0.001 - torch.bfloat16: - rtol: 0.004 - small_value: 0.001 - small_value_atol: 0.001 - \ No newline at end of file + - mul + - mul_ + - __mul__ + - __imul__ + - __rmul__ + - add + - add_ + - __add__ + - __iadd__ + - __radd__ + - div + - div_ + - __div__ + - __idiv__ + - divide + - divide_ + - leaky_relu + - leaky_relu_ + - prelu + - reciprocal + - reciprocal_ + - rsqrt + - rsqrt_ + - square + - square_ + - sub + - sub_ + - rsub + - __isub__ + - __sub__ + +BinaryCompareStandard: + - abs + - abs_ + - absolute + - absolute_ + - argmin + - bitwise_and + - bitwise_and_ + - broadcast_to + - ceil + - ceil_ + - equal + - fill_ + - flatten + - floor + - floor_ + - gather + - greater + - greater_ + - greater_equal + - greater_equal_ + - isfinite + - isnan + - less + - less_ + - less_equal + - less_equal_ + - logical_and + - logical_and_ + - logical_not + - logical_not_ + - logical_or + - logical_or_ + - masked_fill + - masked_fill_ + - max_pool3d + - maximum + - minimum + - neg + - neg_ + - nonzero + - not_equal + - not_equal_ + - one_hot + - pad + - relu + - reshape + - round + - round_ + - select + - sign + - sign_ + - sort + - tile + - transpose + - transpose_ + - tril + - tril_ + - triu + - triu_ -- Gitee From 6774f9cc114005857f40790d07fcc8822a2a08d1 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Thu, 14 Mar 2024 03:07:32 +0000 Subject: [PATCH 02/17] add debug/accuracy_tools/api_accuracy_checker/compare/api_precision_threshold.yaml. Signed-off-by: jiangchangting1 --- .../compare/api_precision_threshold.yaml | 390 ++++++++++++++++++ 1 file changed, 390 insertions(+) create mode 100644 debug/accuracy_tools/api_accuracy_checker/compare/api_precision_threshold.yaml diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_threshold.yaml b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_threshold.yaml new file mode 100644 index 0000000000..7565112da3 --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_threshold.yaml @@ -0,0 +1,390 @@ +mul: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +mul_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__mul__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__imul__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__rmul__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +add: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +add_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__add__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__iadd__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__radd__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +div: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +div_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__div__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__idiv__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +divide: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +divide_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +leaky_relu: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +leaky_relu_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +prelu: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +reciprocal: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +reciprocal_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +rsqrt: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +rsqrt_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +square: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +square_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +sub: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +sub_: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +rsub: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__isub__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 +__sub__: + torch.float32: + rtol: 0.000001 + small_value: 0.000001 + small_value_atol: 0.000001 + torch.float16: + rtol: 0.001 + small_value: 0.001 + small_value_atol: 0.001 + torch.bfloat16: + rtol: 0.004 + small_value: 0.001 + small_value_atol: 0.001 -- Gitee From 0eaa9674c4dd26a2f4c7cfc2ddb6ad7bc5dac299 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Thu, 14 Mar 2024 03:08:31 +0000 Subject: [PATCH 03/17] update debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/algorithm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py index bca77a887d..9dd204f5bc 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/algorithm.py @@ -10,7 +10,7 @@ def cosine_sim(bench_output, device_output): msg = "" n_value = device_output.reshape(-1) b_value = bench_output.reshape(-1) - cos = CompareConst.NA + cos = CompareConst.SPACE np.seterr(divide="ignore", invalid="ignore") if n_value.shape != b_value.shape: msg = f"Shape of device and bench outputs don't match. device: {n_value.shape}, bench: {b_value.shape}." @@ -25,10 +25,10 @@ def cosine_sim(bench_output, device_output): return cos, True, msg elif n_value_max <= np.finfo(float).eps: msg = "All the data is zero in device dump data." - return CompareConst.NA, False, msg + return CompareConst.SPACE, False, msg elif b_value_max <= np.finfo(float).eps: msg = "All the data is zero in bench dump data." - return CompareConst.NA, False, msg + return CompareConst.SPACE, False, msg else: n_value = n_value.astype(float) / n_value_max b_value = b_value.astype(float) / b_value_max -- Gitee From 90e6d903c6bc4d55e49d29d95163f1bf4d90a74f Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Thu, 14 Mar 2024 03:09:13 +0000 Subject: [PATCH 04/17] update debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py. Signed-off-by: jiangchangting1 --- .../compare/api_precision_compare.py | 59 ++++++++++++++----- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index b86402d05a..3c497c74ef 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -11,7 +11,7 @@ from api_accuracy_checker.common.utils import print_info_log, print_warn_log, pr from api_accuracy_checker.common.config import msCheckerConfig from api_accuracy_checker.compare.compare_utils import CompareConst, API_PRECISION_COMPARE_RESULT_FILE_NAME, \ API_PRECISION_COMPARE_DETAILS_FILE_NAME, BENCHMARK_COMPARE_SUPPORT_LIST, API_PRECISION_COMPARE_UNSUPPORT_LIST, \ - ApiPrecisionCompareColumn, AbsoluteStandardApiName, BINARY_COMPARE_UNSUPPORT_LIST + ApiPrecisionCompareColumn, AbsoluteStandardApi, BinaryStandardApi, BINARY_COMPARE_UNSUPPORT_LIST from api_accuracy_checker.compare.compare_column import ApiPrecisionOutputColumn from api_accuracy_checker.run_ut.run_ut import get_validated_result_csv_path from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker, change_mode @@ -113,6 +113,7 @@ class BenchmarkStandard: @staticmethod def _calc_ratio(x, y, default_value=1.0): + x, y = float(x), float(y) if math.isclose(y, 0.0): return 1.0 if math.isclose(x, 0.0) else default_value else: @@ -170,14 +171,14 @@ def analyse_csv(npu_data, gpu_data, config): raise CompareException(CompareException.INVALID_DATA_ERROR, msg) row_gpu = row_gpu.iloc[0] #当前API的输出为空（例如反向过程中requires_grad=False）,跳过比对 - if pd.isna(row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE]): + if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE].isspace(): continue - _, dedicated_api_name, _ = full_api_name.split("*") - new_status = CompareConst.NA + _, api_name, _ = full_api_name.split("*") + new_status = CompareConst.SPACE compare_column.api_name = full_api_name_with_direction_status - if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in BINARY_COMPARE_UNSUPPORT_LIST: + if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in BINARY_COMPARE_UNSUPPORT_LIST or api_name in BinaryStandardApi: new_status = record_binary_consistency_result(compare_column, row_npu) - elif dedicated_api_name in AbsoluteStandardApiName: + elif api_name in AbsoluteStandardApi: new_status = record_absolute_threshold_result(compare_column, row_npu) elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] in BENCHMARK_COMPARE_SUPPORT_LIST: bs = BenchmarkStandard(full_api_name_with_direction_status, row_npu, row_gpu) @@ -222,13 +223,13 @@ def analyse_csv(npu_data, gpu_data, config): def check_error_rate(npu_error_rate): - return CompareConst.PASS if npu_error_rate == 0 else CompareConst.ERROR + return CompareConst.PASS if float(npu_error_rate) == 0 else CompareConst.ERROR def get_absolute_threshold_result(row_npu): - inf_nan_error_ratio = row_npu[ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO] - rel_err_ratio = row_npu[ApiPrecisionCompareColumn.REL_ERR_RATIO] - abs_err_ratio = row_npu[ApiPrecisionCompareColumn.ABS_ERR_RATIO] + inf_nan_error_ratio = float(row_npu[ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO]) + rel_err_ratio = float(row_npu[ApiPrecisionCompareColumn.REL_ERR_RATIO]) + abs_err_ratio = float(row_npu[ApiPrecisionCompareColumn.ABS_ERR_RATIO]) inf_nan_result = CompareConst.PASS if inf_nan_error_ratio == 0 else CompareConst.ERROR rel_err_result = CompareConst.PASS if rel_err_ratio == 0 else CompareConst.ERROR @@ -252,7 +253,7 @@ def get_absolute_threshold_result(row_npu): def get_api_checker_result(status): if not status: - return CompareConst.NA + return CompareConst.SPACE for const in (CompareConst.ERROR, CompareConst.WARNING): if const in status: return const @@ -272,7 +273,11 @@ def record_binary_consistency_result(compare_column, row_npu): compare_column.error_rate = row_npu[ApiPrecisionCompareColumn.ERROR_RATE] compare_column.error_rate_status = new_status compare_column.compare_result = new_status - compare_column.algorithm = "二进制一致法" + compare_column.compare_algorithm = "二进制一致法" + message = '' + if compare_column.error_rate_status == CompareConst.ERROR: + message += "ERROR: 二进制一致错误率超过阈值" + compare_column.compare_message = message return new_status @@ -285,7 +290,15 @@ def record_absolute_threshold_result(compare_column, row_npu): compare_column.abs_err_ratio = absolute_threshold_result.get("abs_err_ratio") compare_column.abs_err_ratio_status = absolute_threshold_result.get("abs_err_result") compare_column.compare_result = absolute_threshold_result.get("absolute_threshold_result") - compare_column.algorithm = "绝对阈值法" + compare_column.compare_algorithm = "绝对阈值法" + message = '' + if compare_column.inf_nan_error_ratio_status == CompareConst.ERROR: + message += "ERROR: inf/nan错误率超过阈值\n" + if compare_column.rel_err_ratio_status == CompareConst.ERROR: + message += "ERROR: 相对误差错误率超过阈值\n" + if compare_column.abs_err_ratio_status == CompareConst.ERROR: + message += "ERROR: 绝对误差错误率超过阈值\n" + compare_column.compare_message = message return compare_column.compare_result @@ -302,7 +315,25 @@ def record_benchmark_compare_result(compare_column, bs): compare_column.eb_ratio = bs.eb_ratio compare_column.eb_status = bs.eb_status compare_column.compare_result = bs.final_result - compare_column.algorithm = "标杆比对法" + compare_column.compare_algorithm = "标杆比对法" + message = '' + if compare_column.small_value_err_status == CompareConst.ERROR: + message += "ERROR: 小值域错误比值超过阈值\n" + if compare_column.rmse_status == CompareConst.ERROR: + message += "ERROR: 均方根误差比值超过阈值\n" + if compare_column.max_rel_err_status == CompareConst.ERROR: + message += "ERROR: 相对误差最大值比值超过阈值\n" + if compare_column.mean_rel_err_status == CompareConst.ERROR: + message += "ERROR: 相对误差平均值比值超过阈值\n" + if compare_column.small_value_err_status == CompareConst.WARNING: + message += "WARNING: 小值域错误比值超过阈值\n" + if compare_column.rmse_status == CompareConst.WARNING: + message += "WARNING: 均方根误差比值超过阈值\n" + if compare_column.max_rel_err_status == CompareConst.WARNING: + message += "WARNING: 相对误差最大值比值超过阈值\n" + if compare_column.mean_rel_err_status == CompareConst.WARNING: + message += "WARNING: 相对误差平均值比值超过阈值\n" + compare_column.compare_message = message return compare_column.compare_result -- Gitee From 0062b16f23439fb2b2f86b3447dabb6588e1f12e Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Thu, 14 Mar 2024 03:09:35 +0000 Subject: [PATCH 05/17] update debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py. Signed-off-by: jiangchangting1 --- .../compare/compare_column.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py index a1b2346750..961fce6811 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_column.py @@ -3,23 +3,23 @@ from api_accuracy_checker.compare.compare_utils import CompareConst class CompareColumn: def __init__(self): - self.bench_type = CompareConst.NA - self.npu_type = CompareConst.NA - self.shape = CompareConst.NA - self.cosine_sim = CompareConst.NA - self.max_abs_err = CompareConst.NA - self.rel_err_hundredth = CompareConst.NA - self.rel_err_thousandth = CompareConst.NA - self.rel_err_ten_thousandth = CompareConst.NA - self.error_rate = CompareConst.NA - self.EB = CompareConst.NA - self.RMSE = CompareConst.NA - self.small_value_err_ratio = CompareConst.NA - self.Max_rel_error = CompareConst.NA - self.Mean_rel_error = CompareConst.NA - self.inf_nan_error_ratio = CompareConst.NA - self.rel_err_ratio = CompareConst.NA - self.abs_err_ratio = CompareConst.NA + self.bench_type = CompareConst.SPACE + self.npu_type = CompareConst.SPACE + self.shape = CompareConst.SPACE + self.cosine_sim = CompareConst.SPACE + self.max_abs_err = CompareConst.SPACE + self.rel_err_hundredth = CompareConst.SPACE + self.rel_err_thousandth = CompareConst.SPACE + self.rel_err_ten_thousandth = CompareConst.SPACE + self.error_rate = CompareConst.SPACE + self.EB = CompareConst.SPACE + self.RMSE = CompareConst.SPACE + self.small_value_err_ratio = CompareConst.SPACE + self.Max_rel_error = CompareConst.SPACE + self.Mean_rel_error = CompareConst.SPACE + self.inf_nan_error_ratio = CompareConst.SPACE + self.rel_err_ratio = CompareConst.SPACE + self.abs_err_ratio = CompareConst.SPACE def to_column_value(self, is_pass, message): return [self.bench_type, self.npu_type, self.shape, self.cosine_sim, self.max_abs_err, self.rel_err_hundredth, -- Gitee From f7eff2942d1b5d5855d32a8dd8abaac78b48db5b Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Thu, 14 Mar 2024 03:09:58 +0000 Subject: [PATCH 06/17] update debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/compare_utils.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 34e4a6ba56..c8c62c1f3b 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -16,11 +16,16 @@ BINARY_COMPARE_UNSUPPORT_LIST = BENCHMARK_COMPARE_SUPPORT_LIST + API_PRECISION_C cur_path = os.path.dirname(os.path.realpath(__file__)) -yaml_path = os.path.join(cur_path, "api_precision_standard.yaml") -with FileOpen(yaml_path, 'r') as f: +standard_yaml_path = os.path.join(cur_path, "api_precision_standard.yaml") +with FileOpen(standard_yaml_path, 'r') as f: Apis = yaml.safe_load(f) AbsoluteStandardApi = Apis.get('AbsoluteThreshStandard') - AbsoluteStandardApiName = list(AbsoluteStandardApi.keys()) + BinaryStandardApi = Apis.get('BinaryCompareStandard') + + +threshold_yaml_path = os.path.join(cur_path, "api_precision_threshold.yaml") +with FileOpen(threshold_yaml_path, 'r') as f: + apis_threshold = yaml.safe_load(f) DETAIL_TEST_ROWS = [[ -- Gitee From 651439f7142caeb87584a2dcc9244acef10f784e Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Thu, 14 Mar 2024 03:12:35 +0000 Subject: [PATCH 07/17] update debug/accuracy_tools/api_accuracy_checker/compare/compare.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/compare.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 1906d6bbee..84e2e70ff0 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -7,7 +7,7 @@ from rich.table import Table from rich.console import Console from api_accuracy_checker.common.utils import get_json_contents, write_csv from api_accuracy_checker.compare.compare_utils import CompareConst, check_dtype_comparable, DETAIL_TEST_ROWS, \ - precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, AbsoluteStandardApi, AbsoluteStandardApiName + precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, AbsoluteStandardApi, BinaryStandardApi, apis_threshold from api_accuracy_checker.compare.compare_column import CompareColumn from api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, get_mean_rel_err, \ get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \ @@ -72,7 +72,7 @@ class Comparator: console.print(table_detail) def get_statistics_from_result_csv(self): - checklist = [CompareConst.PASS, CompareConst.ERROR, CompareConst.WARNING, CompareConst.NA, CompareConst.SKIP, "skip"] + checklist = [CompareConst.PASS, CompareConst.ERROR, CompareConst.WARNING, CompareConst.SPACE, CompareConst.SKIP, "skip"] self.test_result_cnt = { "success_num": 0, "warning_num": 0, "error_num": 0, "forward_fail_num": 0, "backward_fail_num": 0, "forward_and_backward_fail_num": 0, @@ -95,7 +95,7 @@ class Comparator: self.test_result_cnt["total_skip_num"] += 1 continue self.test_result_cnt["total_num"] += 1 - if column1 == CompareConst.PASS and column2 in [CompareConst.PASS, CompareConst.NA]: + if column1 == CompareConst.PASS and column2 in [CompareConst.PASS, CompareConst.SPACE]: self.test_result_cnt['success_num'] += 1 elif column1 == CompareConst.ERROR and column2 == CompareConst.ERROR: self.test_result_cnt['forward_and_backward_fail_num'] += 1 @@ -161,7 +161,7 @@ class Comparator: compare_func = self._compare_dropout if "dropout" in full_api_name else self._compare_core_wrapper fwd_success_status, fwd_compare_alg_results = compare_func(api_name, bench_output, device_output) bwd_success_status, bwd_compare_alg_results = (CompareConst.PASS, []) if not (bench_grad and npu_grad) else compare_func(api_name, bench_grad[0], npu_grad[0]) if "dropout" in full_api_name else compare_func(api_name, bench_grad, npu_grad) - self.record_results(full_api_name, fwd_success_status, bwd_success_status if bwd_compare_alg_results is not None else CompareConst.NA, fwd_compare_alg_results, bwd_compare_alg_results) + self.record_results(full_api_name, fwd_success_status, bwd_success_status if bwd_compare_alg_results is not None else CompareConst.SPACE, fwd_compare_alg_results, bwd_compare_alg_results) return fwd_success_status == CompareConst.PASS, bwd_success_status == CompareConst.PASS def _compare_core_wrapper(self, api_name, bench_output, device_output): @@ -261,7 +261,10 @@ class Comparator: abs_err = get_abs_err(bench_output, device_output) if str(dtype) in BENCHMARK_COMPARE_SUPPORT_LIST: both_finite_mask, inf_nan_mask = get_finite_and_infinite_mask(bench_output, device_output) - if api_name in AbsoluteStandardApiName: + if api_name in BinaryStandardApi: + err_rate, _, _ = self._compare_bool_tensor(bench_output, device_output) + compare_column.error_rate = err_rate + elif api_name in AbsoluteStandardApi: small_value_threshold, small_value_atol, rtol = self._get_absolute_threshold_attribute( api_name, str(dtype)) rel_err = abs_err / abs_bench_with_eps @@ -353,7 +356,7 @@ class Comparator: @staticmethod def _get_absolute_threshold_attribute(api_name, dtype): - small_value_threshold = AbsoluteStandardApi.get(api_name).get(dtype).get('small_value') - small_value_atol = AbsoluteStandardApi.get(api_name).get(dtype).get('small_value_atol') - rtol = AbsoluteStandardApi.get(api_name).get(dtype).get('rtol') + small_value_threshold = apis_threshold.get(api_name).get(dtype).get('small_value') + small_value_atol = apis_threshold.get(api_name).get(dtype).get('small_value_atol') + rtol = apis_threshold.get(api_name).get(dtype).get('rtol') return small_value_threshold, small_value_atol, rtol -- Gitee From 5680713140bfe840ab14d499d63e6482eec23520 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Fri, 15 Mar 2024 03:00:39 +0000 Subject: [PATCH 08/17] update debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py. Signed-off-by: jiangchangting1 --- .../compare/compare_utils.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index c8c62c1f3b..327b2f4f15 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -167,3 +167,19 @@ def check_dtype_comparable(x, y): return False print_warn_log(f"Compare: Unexpected dtype {x.dtype}, {y.dtype}") return False + + +def save_float_convert(input_data): + if input_data.strip() == "": + msg = 'ERROR: Input data is an empty string' + raise CompareException(CompareException.INVALID_DATA_ERROR, msg) + try: + float_data = float(input_data) + if float_data == float('inf') or float_data == float('-inf') or float_data != float_data: + msg = 'ERROR: Input data is either "inf", "-inf", "nan"' + raise CompareException(CompareException.INVALID_DATA_ERROR, msg) + return float_data + except ValueError: + msg = 'ERROR: Input data cannot be converted to float' + print_error_log(msg) + \ No newline at end of file -- Gitee From 03627cf36c57792ce7bdc250703429a95afb9167 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Fri, 15 Mar 2024 03:02:47 +0000 Subject: [PATCH 09/17] update debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py. Signed-off-by: jiangchangting1 --- .../compare/api_precision_compare.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index 3c497c74ef..6a2f7557ea 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -11,7 +11,7 @@ from api_accuracy_checker.common.utils import print_info_log, print_warn_log, pr from api_accuracy_checker.common.config import msCheckerConfig from api_accuracy_checker.compare.compare_utils import CompareConst, API_PRECISION_COMPARE_RESULT_FILE_NAME, \ API_PRECISION_COMPARE_DETAILS_FILE_NAME, BENCHMARK_COMPARE_SUPPORT_LIST, API_PRECISION_COMPARE_UNSUPPORT_LIST, \ - ApiPrecisionCompareColumn, AbsoluteStandardApi, BinaryStandardApi, BINARY_COMPARE_UNSUPPORT_LIST + ApiPrecisionCompareColumn, AbsoluteStandardApi, BinaryStandardApi, BINARY_COMPARE_UNSUPPORT_LIST, save_float_convert from api_accuracy_checker.compare.compare_column import ApiPrecisionOutputColumn from api_accuracy_checker.run_ut.run_ut import get_validated_result_csv_path from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker, change_mode @@ -113,7 +113,7 @@ class BenchmarkStandard: @staticmethod def _calc_ratio(x, y, default_value=1.0): - x, y = float(x), float(y) + x, y = save_float_convert(x), save_float_convert(y) if math.isclose(y, 0.0): return 1.0 if math.isclose(x, 0.0) else default_value else: @@ -223,13 +223,13 @@ def analyse_csv(npu_data, gpu_data, config): def check_error_rate(npu_error_rate): - return CompareConst.PASS if float(npu_error_rate) == 0 else CompareConst.ERROR + return CompareConst.PASS if save_float_convert(npu_error_rate) == 0 else CompareConst.ERROR def get_absolute_threshold_result(row_npu): - inf_nan_error_ratio = float(row_npu[ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO]) - rel_err_ratio = float(row_npu[ApiPrecisionCompareColumn.REL_ERR_RATIO]) - abs_err_ratio = float(row_npu[ApiPrecisionCompareColumn.ABS_ERR_RATIO]) + inf_nan_error_ratio = save_float_convert(row_npu[ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO]) + rel_err_ratio = save_float_convert(row_npu[ApiPrecisionCompareColumn.REL_ERR_RATIO]) + abs_err_ratio = save_float_convert(row_npu[ApiPrecisionCompareColumn.ABS_ERR_RATIO]) inf_nan_result = CompareConst.PASS if inf_nan_error_ratio == 0 else CompareConst.ERROR rel_err_result = CompareConst.PASS if rel_err_ratio == 0 else CompareConst.ERROR -- Gitee From cec37fa6cd2749940ffb68f99c1f0c03e3c35dec Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Fri, 15 Mar 2024 03:04:28 +0000 Subject: [PATCH 10/17] update debug/accuracy_tools/api_accuracy_checker/compare/compare.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/compare.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py index 84e2e70ff0..15bfb1904c 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare.py @@ -87,7 +87,7 @@ class Comparator: raise ValueError("The number of columns in %s is incorrect" % result_csv_name) if not all(item[i] and item[i] in checklist for i in (1, 2)): raise ValueError( - "The value in the 2nd or 3rd column of %s is wrong, it must be pass, error, warning, skip, or N/A" + "The value in the 2nd or 3rd column of %s is wrong, it must be pass, error, warning, skip, or SPACE" % result_csv_name) column1 = item[1] column2 = item[2] @@ -288,13 +288,13 @@ class Comparator: compare_column.cosine_sim = cos_res message += msg + "\n" if not cos_status: - message += "Cosine similarity is less than 0.99, consider as error, skip other check and set to N/A.\n" + message += "Cosine similarity is less than 0.99, consider as error, skip other check and set to SPACE.\n" return CompareConst.ERROR, compare_column, message max_abs_res, max_abs_status = get_max_abs_err(abs_err) compare_column.max_abs_err = max_abs_res if max_abs_status: - message += "Max abs error is less than 0.001, consider as pass, skip other check and set to N/A.\n" + message += "Max abs error is less than 0.001, consider as pass, skip other check and set to SPACE.\n" return CompareConst.PASS, compare_column, message rel_err_orign = get_rel_err_origin(abs_err, abs_bench_with_eps) @@ -302,24 +302,24 @@ class Comparator: hundred_res, hundred_status = get_rel_err_ratio(rel_err_orign, 0.01) compare_column.rel_err_hundredth = hundred_res if not hundred_status: - message += "Relative error is greater than 0.01, consider as error, skip other check and set to N/A.\n" + message += "Relative error is greater than 0.01, consider as error, skip other check and set to SPACE.\n" return CompareConst.ERROR, compare_column, message thousand_res, thousand_status = get_rel_err_ratio(rel_err_orign, 0.001) compare_column.rel_err_thousandth = thousand_res if dtype in [torch.float16, torch.bfloat16]: if thousand_status: - message += "Relative error is less than 0.001, consider as pass, skip other check and set to N/A.\n" + message += "Relative error is less than 0.001, consider as pass, skip other check and set to SPACE.\n" return CompareConst.PASS, compare_column, message - message += "Relative error is greater than 0.001, consider as warning, skip other check and set to N/A.\n" + message += "Relative error is greater than 0.001, consider as warning, skip other check and set to SPACE.\n" return CompareConst.WARNING, compare_column, message ten_thousand_res, ten_thousand_status = get_rel_err_ratio(rel_err_orign, 0.0001) compare_column.rel_err_ten_thousandth = ten_thousand_res if dtype in [torch.float32, torch.float64]: if not thousand_status: - message += "Relative error is greater than 0.001, consider as error, skip other check and set to N/A.\n" + message += "Relative error is greater than 0.001, consider as error, skip other check and set to SPACE.\n" return CompareConst.ERROR, compare_column, message if not ten_thousand_status: - message += "Relative error is greater than 0.0001, consider as warning, skip other check and set to N/A.\n" + message += "Relative error is greater than 0.0001, consider as warning, skip other check and set to SPACE.\n" return CompareConst.WARNING, compare_column, message message += "Relative error is less than 0.0001, consider as pass.\n" return CompareConst.PASS, compare_column, message -- Gitee From 90183f3cc8ef1a4925c92f77d7688b2dfed7e84d Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Fri, 15 Mar 2024 09:42:28 +0000 Subject: [PATCH 11/17] update debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/compare_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 327b2f4f15..65eadbabfd 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -181,5 +181,5 @@ def save_float_convert(input_data): return float_data except ValueError: msg = 'ERROR: Input data cannot be converted to float' - print_error_log(msg) + raise CompareException(CompareException.INVALID_DATA_ERROR, msg) \ No newline at end of file -- Gitee From a81e160e60b126bcd5885e8d719f329ad33326ec Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 18 Mar 2024 06:18:56 +0000 Subject: [PATCH 12/17] update debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/compare_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 65eadbabfd..42ae7c96ff 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -179,7 +179,7 @@ def save_float_convert(input_data): msg = 'ERROR: Input data is either "inf", "-inf", "nan"' raise CompareException(CompareException.INVALID_DATA_ERROR, msg) return float_data - except ValueError: + except ValueError as e: msg = 'ERROR: Input data cannot be converted to float' - raise CompareException(CompareException.INVALID_DATA_ERROR, msg) + raise CompareException(CompareException.INVALID_DATA_ERROR, msg) from e \ No newline at end of file -- Gitee From a78b2af4a227b4fc8858f1d0e02b5027663df166 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 18 Mar 2024 09:34:54 +0000 Subject: [PATCH 13/17] update debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/compare_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 42ae7c96ff..fdde4566d7 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -170,7 +170,7 @@ def check_dtype_comparable(x, y): def save_float_convert(input_data): - if input_data.strip() == "": + if isinstance(input_data, str) and input_data.strip() == "": msg = 'ERROR: Input data is an empty string' raise CompareException(CompareException.INVALID_DATA_ERROR, msg) try: -- Gitee From 79b78275ad5b0badaec43d392157bdf995e61db5 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Tue, 19 Mar 2024 13:36:03 +0000 Subject: [PATCH 14/17] update debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/compare_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index fdde4566d7..5c91e2fa8c 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -175,7 +175,7 @@ def save_float_convert(input_data): raise CompareException(CompareException.INVALID_DATA_ERROR, msg) try: float_data = float(input_data) - if float_data == float('inf') or float_data == float('-inf') or float_data != float_data: + if str(float_data) in ('inf', '-inf', 'nan'): msg = 'ERROR: Input data is either "inf", "-inf", "nan"' raise CompareException(CompareException.INVALID_DATA_ERROR, msg) return float_data -- Gitee From 763b1c4e4a2e87ecafe6b7f3ce55ace65b94fcb4 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Wed, 20 Mar 2024 01:16:28 +0000 Subject: [PATCH 15/17] update debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py. Signed-off-by: jiangchangting1 --- .../compare/api_precision_compare.py | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index 6a2f7557ea..9cde5b48d7 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -46,6 +46,26 @@ benchmark_algorithms_thresholds = { } +benchmark_message = { + "small_value_err_status": { + CompareConst.ERROR: "ERROR: 小值域错误比值超过阈值\n", + CompareConst.WARNING: "WARNING: 小值域错误比值超过阈值\n" + }, + "rmse_status": { + CompareConst.ERROR: "ERROR: 均方根误差比值超过阈值\n", + CompareConst.WARNING: "WARNING: 均方根误差比值超过阈值\n" + }, + "max_rel_err_status": { + CompareConst.ERROR: "ERROR: 相对误差最大值比值超过阈值\n", + CompareConst.WARNING: "WARNING: 相对误差最大值比值超过阈值\n" + }, + "mean_rel_err_status": { + CompareConst.ERROR: "ERROR: 相对误差平均值比值超过阈值\n", + CompareConst.WARNING: "WARNING: 相对误差平均值比值超过阈值\n" + } +} + + class BenchmarkStandard: def __init__(self, api_name, npu_precision, gpu_precision): self.api_name = api_name @@ -317,22 +337,10 @@ def record_benchmark_compare_result(compare_column, bs): compare_column.compare_result = bs.final_result compare_column.compare_algorithm = "标杆比对法" message = '' - if compare_column.small_value_err_status == CompareConst.ERROR: - message += "ERROR: 小值域错误比值超过阈值\n" - if compare_column.rmse_status == CompareConst.ERROR: - message += "ERROR: 均方根误差比值超过阈值\n" - if compare_column.max_rel_err_status == CompareConst.ERROR: - message += "ERROR: 相对误差最大值比值超过阈值\n" - if compare_column.mean_rel_err_status == CompareConst.ERROR: - message += "ERROR: 相对误差平均值比值超过阈值\n" - if compare_column.small_value_err_status == CompareConst.WARNING: - message += "WARNING: 小值域错误比值超过阈值\n" - if compare_column.rmse_status == CompareConst.WARNING: - message += "WARNING: 均方根误差比值超过阈值\n" - if compare_column.max_rel_err_status == CompareConst.WARNING: - message += "WARNING: 相对误差最大值比值超过阈值\n" - if compare_column.mean_rel_err_status == CompareConst.WARNING: - message += "WARNING: 相对误差平均值比值超过阈值\n" + for status_attr, messages in benchmark_message.items(): + status_value = getattr(compare_column, status_attr) + if status_value in messages: + message += messages[status_value] compare_column.compare_message = message return compare_column.compare_result -- Gitee From 6dc8147e212d73cc1082dea2a3229c2c9cc2bcf0 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Wed, 20 Mar 2024 01:47:46 +0000 Subject: [PATCH 16/17] update debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py. Signed-off-by: jiangchangting1 --- .../compare/api_precision_compare.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py index 9cde5b48d7..8c98130ab6 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/api_precision_compare.py @@ -11,7 +11,7 @@ from api_accuracy_checker.common.utils import print_info_log, print_warn_log, pr from api_accuracy_checker.common.config import msCheckerConfig from api_accuracy_checker.compare.compare_utils import CompareConst, API_PRECISION_COMPARE_RESULT_FILE_NAME, \ API_PRECISION_COMPARE_DETAILS_FILE_NAME, BENCHMARK_COMPARE_SUPPORT_LIST, API_PRECISION_COMPARE_UNSUPPORT_LIST, \ - ApiPrecisionCompareColumn, AbsoluteStandardApi, BinaryStandardApi, BINARY_COMPARE_UNSUPPORT_LIST, save_float_convert + ApiPrecisionCompareColumn, AbsoluteStandardApi, BinaryStandardApi, BINARY_COMPARE_UNSUPPORT_LIST, convert_str_to_float from api_accuracy_checker.compare.compare_column import ApiPrecisionOutputColumn from api_accuracy_checker.run_ut.run_ut import get_validated_result_csv_path from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker, change_mode @@ -133,7 +133,7 @@ class BenchmarkStandard: @staticmethod def _calc_ratio(x, y, default_value=1.0): - x, y = save_float_convert(x), save_float_convert(y) + x, y = convert_str_to_float(x), convert_str_to_float(y) if math.isclose(y, 0.0): return 1.0 if math.isclose(x, 0.0) else default_value else: @@ -243,13 +243,13 @@ def analyse_csv(npu_data, gpu_data, config): def check_error_rate(npu_error_rate): - return CompareConst.PASS if save_float_convert(npu_error_rate) == 0 else CompareConst.ERROR + return CompareConst.PASS if convert_str_to_float(npu_error_rate) == 0 else CompareConst.ERROR def get_absolute_threshold_result(row_npu): - inf_nan_error_ratio = save_float_convert(row_npu[ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO]) - rel_err_ratio = save_float_convert(row_npu[ApiPrecisionCompareColumn.REL_ERR_RATIO]) - abs_err_ratio = save_float_convert(row_npu[ApiPrecisionCompareColumn.ABS_ERR_RATIO]) + inf_nan_error_ratio = convert_str_to_float(row_npu[ApiPrecisionCompareColumn.INF_NAN_ERROR_RATIO]) + rel_err_ratio = convert_str_to_float(row_npu[ApiPrecisionCompareColumn.REL_ERR_RATIO]) + abs_err_ratio = convert_str_to_float(row_npu[ApiPrecisionCompareColumn.ABS_ERR_RATIO]) inf_nan_result = CompareConst.PASS if inf_nan_error_ratio == 0 else CompareConst.ERROR rel_err_result = CompareConst.PASS if rel_err_ratio == 0 else CompareConst.ERROR -- Gitee From fecce4cdd3b7c08b2d85239d382dca2ee0cf88b9 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Wed, 20 Mar 2024 01:48:44 +0000 Subject: [PATCH 17/17] update debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/compare_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py index 5c91e2fa8c..d711265cc7 100644 --- a/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/compare/compare_utils.py @@ -169,7 +169,7 @@ def check_dtype_comparable(x, y): return False -def save_float_convert(input_data): +def convert_str_to_float(input_data): if isinstance(input_data, str) and input_data.strip() == "": msg = 'ERROR: Input data is an empty string' raise CompareException(CompareException.INVALID_DATA_ERROR, msg) -- Gitee