From 8cb4bdb8059f9607e50c742a86cf2ea7e06df609 Mon Sep 17 00:00:00 2001 From: wangchao Date: Tue, 18 Jul 2023 12:20:07 +0000 Subject: [PATCH 01/19] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20api=5Fut=5Ftools?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/tools/api_ut_tools/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 debug/tools/api_ut_tools/.keep diff --git a/debug/tools/api_ut_tools/.keep b/debug/tools/api_ut_tools/.keep new file mode 100644 index 00000000000..e69de29bb2d -- Gitee From 52328a9bf02810d53918ce625b0015b568301695 Mon Sep 17 00:00:00 2001 From: wangchao Date: Tue, 18 Jul 2023 12:21:03 +0000 Subject: [PATCH 02/19] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20common?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/tools/api_ut_tools/common/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 debug/tools/api_ut_tools/common/.keep diff --git a/debug/tools/api_ut_tools/common/.keep b/debug/tools/api_ut_tools/common/.keep new file mode 100644 index 00000000000..e69de29bb2d -- Gitee From 95dd227a0ac2d03d5c4e11abcf3cd56011741b4f Mon Sep 17 00:00:00 2001 From: wangchao Date: Tue, 18 Jul 2023 12:21:28 +0000 Subject: [PATCH 03/19] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20dump?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/tools/api_ut_tools/dump/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 debug/tools/api_ut_tools/dump/.keep diff --git a/debug/tools/api_ut_tools/dump/.keep b/debug/tools/api_ut_tools/dump/.keep new file mode 100644 index 00000000000..e69de29bb2d -- Gitee From 70eafa39791f4a5d9754fd325f78e2ff14f38747 Mon Sep 17 00:00:00 2001 From: wangchao Date: Tue, 18 Jul 2023 12:21:44 +0000 Subject: [PATCH 04/19] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20hook=5Fmodule?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/tools/api_ut_tools/hook_module/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 debug/tools/api_ut_tools/hook_module/.keep diff --git a/debug/tools/api_ut_tools/hook_module/.keep b/debug/tools/api_ut_tools/hook_module/.keep new file mode 100644 index 00000000000..e69de29bb2d -- Gitee From 7e3915858b7b6043106022d0199672199b2a985b Mon Sep 17 00:00:00 2001 From: wangchao Date: Tue, 18 Jul 2023 12:22:09 +0000 Subject: [PATCH 05/19] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20run=5Fut?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/tools/api_ut_tools/run_ut/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 debug/tools/api_ut_tools/run_ut/.keep diff --git a/debug/tools/api_ut_tools/run_ut/.keep b/debug/tools/api_ut_tools/run_ut/.keep new file mode 100644 index 00000000000..e69de29bb2d -- Gitee From 0e383fc72ad7d733f54e372fc339cd507f3ad2e1 Mon Sep 17 00:00:00 2001 From: wangchao Date: Tue, 18 Jul 2023 12:23:55 +0000 Subject: [PATCH 06/19] add debug/tools/api_ut_tools/hook_module/hook_module.py. Signed-off-by: wangchao --- .../api_ut_tools/hook_module/hook_module.py | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 debug/tools/api_ut_tools/hook_module/hook_module.py diff --git a/debug/tools/api_ut_tools/hook_module/hook_module.py b/debug/tools/api_ut_tools/hook_module/hook_module.py new file mode 100644 index 00000000000..c18b37a9c9a --- /dev/null +++ b/debug/tools/api_ut_tools/hook_module/hook_module.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + + +import functools + +import torch +import torch.nn as nn +import torch.utils.hooks as full_hooks + +module_count = {} +g_stop_hook = False + + +class HOOKModule(nn.Module): + + def __init__(self, hook) -> None: + super(HOOKModule, self).__init__() + self.has_overflow = False + self.input_args = tuple() + self.input_kwargs = dict() + self._enable_hook = True + prefix = "" + if hasattr(self, "prefix_op_name_"): + prefix = self.prefix_op_name_ + + if prefix not in module_count: + module_count[prefix] = 1 + prefix += '0*' + else: + module_count[prefix] += 1 + prefix = prefix + str(module_count[prefix] - 1) + '*' + + self.register_forward_hook(hook(prefix + "forward")) + self.register_backward_hook(hook(prefix + "backward")) + + def __call__(self, *input, **kwargs): + changed = False + global g_stop_hook + if g_stop_hook: + self._enable_hook = False + else: + g_stop_hook = True + changed = True + result = self._call_func(*input, **kwargs) + if changed: + g_stop_hook = False + return result + + def _call_func(self, *input, **kwargs): + if self._enable_hook: + full_backward_hooks, non_full_backward_hooks = [], [] + if len(self._backward_hooks) > 0: + full_backward_hooks, non_full_backward_hooks = self._get_backward_hooks() + for hook in self._forward_pre_hooks.values(): + result = hook(self, input) + if result is not None: + if not isinstance(result, tuple): + result = (result,) + input = result + bw_hook = None + if len(full_backward_hooks) > 0: + bw_hook = full_hooks.BackwardHook(self, full_backward_hooks) + input = bw_hook.setup_input_hook(input) + self.input_args = input + self.input_kwargs = kwargs + if torch._C._get_tracing_state(): + result = self._slow_forward(*input, **kwargs) + else: + result = self.forward(*input, **kwargs) + for hook in self._forward_hooks.values(): + hook_result = hook(self, input, result) + if hook_result is not None: + result = hook_result + if bw_hook: + result = bw_hook.setup_output_hook(result) + if len(non_full_backward_hooks) > 0: + var = result + while not isinstance(var, torch.Tensor): + if isinstance(var, dict): + var = next((v for v in var.values() if isinstance(v, torch.Tensor))) + elif isinstance(var, (list, tuple)): + if var: + var = var[0] + else: + return result + else: + return result + grad_fn = var.grad_fn + if grad_fn is not None: + for hook in non_full_backward_hooks: + wrapper = functools.partial(hook, self) + functools.update_wrapper(wrapper, hook) + grad_fn.register_hook(wrapper) + self._maybe_warn_non_full_backward_hook(input, result, grad_fn) + return result + else: + forward_call = (self._slow_forward if torch._C._get_tracing_state() else self.forward) + return forward_call(*input, **kwargs) -- Gitee From 93b1e994ac8fac78395b899438f4210ee1d75296 Mon Sep 17 00:00:00 2001 From: wangchao Date: Tue, 18 Jul 2023 12:24:54 +0000 Subject: [PATCH 07/19] add debug/tools/api_ut_tools/hook_module/register_hook.py. Signed-off-by: wangchao --- .../api_ut_tools/hook_module/register_hook.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 debug/tools/api_ut_tools/hook_module/register_hook.py diff --git a/debug/tools/api_ut_tools/hook_module/register_hook.py b/debug/tools/api_ut_tools/hook_module/register_hook.py new file mode 100644 index 00000000000..35d02bb641c --- /dev/null +++ b/debug/tools/api_ut_tools/hook_module/register_hook.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +import functools +import os + +import torch + +from . import wrap_torch, wrap_functional, wrap_tensor +from .hook_module import HOOKModule +from ..common.utils import check_file_or_directory_path, print_error_log, CompareException, Const, \ + print_info_log, print_warn_log, get_process_rank +from ..dump.utils import make_dump_dirs +from ..dump.dump import acc_cmp_dump + +try: + import torch_npu +except ImportError: + is_gpu = True +else: + is_gpu = False + +make_dir_flag = True + + +def initialize_hook(hook): + wrap_tensor.wrap_tensor_ops_and_bind(hook) + for attr_name in dir(wrap_tensor.HOOKTensor): + if attr_name.startswith("wrap_"): + setattr(torch.Tensor, attr_name[5:], getattr(wrap_tensor.HOOKTensor, attr_name)) + + wrap_torch.wrap_torch_ops_and_bind(hook) + for attr_name in dir(wrap_torch.HOOKTorchOP): + if attr_name.startswith("wrap_"): + setattr(torch, attr_name[5:], getattr(wrap_torch.HOOKTorchOP, attr_name)) + + wrap_functional.wrap_functional_ops_and_bind(hook) + for attr_name in dir(wrap_functional.HOOKFunctionalOP): + if attr_name.startswith("wrap_"): + setattr(torch.nn.functional, attr_name[5:], getattr(wrap_functional.HOOKFunctionalOP, attr_name)) + + +def register_hook(): + global make_dir_flag + if make_dir_flag: + make_dump_dirs(0) + make_dir_flag = False + initialize_hook(acc_cmp_dump) -- Gitee From bb46b626d5af4375b1b7e7113e751d2bce162e76 Mon Sep 17 00:00:00 2001 From: wangchao Date: Tue, 18 Jul 2023 12:25:46 +0000 Subject: [PATCH 08/19] add debug/tools/api_ut_tools/hook_module/support_wrap_ops.yaml. Signed-off-by: wangchao --- .../hook_module/support_wrap_ops.yaml | 1000 +++++++++++++++++ 1 file changed, 1000 insertions(+) create mode 100644 debug/tools/api_ut_tools/hook_module/support_wrap_ops.yaml diff --git a/debug/tools/api_ut_tools/hook_module/support_wrap_ops.yaml b/debug/tools/api_ut_tools/hook_module/support_wrap_ops.yaml new file mode 100644 index 00000000000..6aefa1460b3 --- /dev/null +++ b/debug/tools/api_ut_tools/hook_module/support_wrap_ops.yaml @@ -0,0 +1,1000 @@ +# Copyright (c) 2020 Huawei Technologies Co., Ltd +# All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# List of ops that register hooks + +functional: + - conv1d + - conv2d + - conv3d + - conv_transpose1d + - conv_transpose2d + - conv_transpose3d + - conv_tbc + - avg_pool1d + - avg_pool2d + - avg_pool3d + - fractional_max_pool2d_with_indices + - fractional_max_pool2d + - fractional_max_pool3d_with_indices + - fractional_max_pool3d + - max_pool1d_with_indices + - max_pool1d + - max_pool2d_with_indices + - max_pool2d + - max_pool3d_with_indices + - max_pool3d + - max_unpool1d + - max_unpool2d + - max_unpool3d + - lp_pool2d + - lp_pool1d + - adaptive_max_pool1d_with_indices + - adaptive_max_pool1d + - adaptive_max_pool2d_with_indices + - adaptive_max_pool2d + - adaptive_max_pool3d_with_indices + - adaptive_max_pool3d + - adaptive_avg_pool1d + - adaptive_avg_pool2d + - adaptive_avg_pool3d + - dropout + - alpha_dropout + - dropout2d + - dropout3d + - feature_alpha_dropout + - threshold + - threshold_ + - relu + - relu_ + - glu + - hardtanh + - hardtanh_ + - relu6 + - elu + - elu_ + - selu + - selu_ + - celu + - celu_ + - leaky_relu + - leaky_relu_ + - prelu + - rrelu + - rrelu_ + - logsigmoid + - gelu + - hardshrink + - tanhshrink + - softsign + - softplus + - softmin + - softmax + - gumbel_softmax + - log_softmax + - softshrink + - tanh + - sigmoid + - hardsigmoid + - linear + - bilinear + - silu + - hardswish + - embedding + - embedding_bag + - batch_norm + - instance_norm + - layer_norm + - group_norm + - local_response_norm + - ctc_loss + - nll_loss + - poisson_nll_loss + - gaussian_nll_loss + - kl_div + - cross_entropy + - binary_cross_entropy + - binary_cross_entropy_with_logits + - smooth_l1_loss + - l1_loss + - mse_loss + - margin_ranking_loss + - hinge_embedding_loss + - multilabel_margin_loss + - soft_margin_loss + - multilabel_soft_margin_loss + - cosine_embedding_loss + - multi_margin_loss + - pixel_shuffle + - pixel_unshuffle + - channel_shuffle + - upsample + - interpolate + - upsample_nearest + - upsample_bilinear + - grid_sample + - affine_grid + - pad + - pairwise_distance + - pdist + - cosine_similarity + - one_hot + - triplet_margin_loss + - triplet_margin_with_distance_loss + - normalize + - unfold + - fold + - multi_head_attention_forward + +tensor: + - __add__ + - __and__ + - __bool__ + - __div__ + - __eq__ + - __ge__ + - __gt__ + - __getitem__ + - __iadd__ + - __iand__ + - __idiv__ + - __ifloordiv__ + - __ilshift__ + - __imod__ + - __imul__ + - __ior__ + - __irshift__ + - __isub__ + - __ixor__ + - __lshift__ + - __matmul__ + - __mod__ + - __mul__ + - __nonzero__ + - __or__ + - __radd__ + - __rmul__ + - __rshift__ + - __sub__ + - __truediv__ + - __xor__ + - abs + - abs_ + - absolute + - absolute_ + - acos + - acos_ + - acosh + - acosh_ + - add + - add_ + - addbmm + - addbmm_ + - addcdiv + - addcdiv_ + - addcmul + - addcmul_ + - addmm + - addmm_ + - addmv + - addmv_ + - addr + - addr_ + - align_as + - align_to + - all + - allclose + - amax + - amin + - angle + - any + - arccos + - arccos_ + - arccosh + - arccosh_ + - arcsin + - arcsin_ + - arcsinh + - arcsinh_ + - arctan + - arctan_ + - arctanh + - arctanh_ + - argmax + - argmin + - argsort + - asin + - asin_ + - asinh + - asinh_ + - atan + - atan2 + - atan2_ + - atan_ + - atanh + - atanh_ + - baddbmm + - baddbmm_ + - bernoulli + - bernoulli_ + - bincount + - bitwise_and + - bitwise_and_ + - bitwise_not + - bitwise_not_ + - bitwise_or + - bitwise_or_ + - bitwise_xor + - bitwise_xor_ + - bmm + - broadcast_to + - cauchy_ + - ceil + - ceil_ + - cholesky + - chunk + - clamp + - cholesky_solve + - cholesky_inverse + - clamp_ + - clamp_max + - clamp_max_ + - clip + - clamp_min + - clamp_min_ + - clip_ + - copysign + - copysign_ + - cos + - cos_ + - cosh + - cosh_ + - count_nonzero + - cummax + - cummin + - cumprod + - cumprod_ + - cumsum + - cumsum_ + - deg2rad + - deg2rad_ + - det + - diag + - diag_embed + - diagflat + - diagonal + - diff + - dist + - digamma + - digamma_ + - div + - div_ + - divide + - divide_ + - dot + - eig + - eq + - eq_ + - erf + - equal + - erf_ + - erfc + - erfc_ + - erfinv + - erfinv_ + - exp + - exp2 + - exp2_ + - expm1 + - exp_ + - expm1_ + - exponential_ + - fill_ + - fix + - fill_diagonal_ + - fix_ + - flip + - fliplr + - flatten + - flipud + - float_power + - float_power_ + - floor + - floor_ + - floor_divide + - floor_divide_ + - fmax + - fmin + - fmod + - fmod_ + - frac + - frac_ + - gather + - gcd + - gcd_ + - ge + - ge_ + - geometric_ + - geqrf + - ger + - greater + - greater_ + - gt + - gt_ + - greater_equal + - greater_equal_ + - hardshrink + - heaviside + - heaviside_ + - histc + - hypot + - hypot_ + - igamma + - igamma_ + - igammac + - igammac_ + - index_add + - index_add_ + - inverse + - index_copy + - index_copy_ + - index_fill + - index_fill_ + - index_put + - index_put_ + - inner + - index_select + - isclose + - isfinite + - isinf + - isnan + - isneginf + - isposinf + - isreal + - kron + - kthvalue + - lcm + - lcm_ + - ldexp + - ldexp_ + - le + - le_ + - lerp + - lerp_ + - where + - less + - less_ + - less_equal + - less_equal_ + - lgamma + - lgamma_ + - log + - log10 + - log10_ + - log1p + - log1p_ + - log2 + - log2_ + - log_ + - log_normal_ + - log_softmax + - logcumsumexp + - logdet + - logaddexp + - logaddexp2 + - logical_and + - logical_and_ + - logical_not + - logit + - logical_not_ + - logical_or + - logical_or_ + - logical_xor + - logical_xor_ + - logit_ + - logsumexp + - lstsq + - lt + - lt_ + - lu_solve + - map2_ + - map_ + - masked_fill + - matmul + - masked_fill_ + - masked_scatter + - masked_scatter_ + - masked_select + - matrix_exp + - max + - maximum + - mean + - matrix_power + - median + - min + - minimum + - mm + - mode + - msort + - mul + - mul_ + - multinomial + - multiply + - multiply_ + - mv + - mvlgamma + - mvlgamma_ + - nansum + - narrow + - narrow_copy + - ne + - ne_ + - neg + - neg_ + - negative + - negative_ + - nonzero + - normal_ + - not_equal + - not_equal_ + - permute + - pinverse + - polygamma + - pow + - pow_ + - polygamma_ + - prelu + - prod + - put_ + - rad2deg + - rad2deg_ + - ravel + - real + - reciprocal + - reciprocal_ + - relu + - relu_ + - remainder + - repeat_interleave + - reshape + - remainder_ + - renorm + - renorm_ + - repeat + - reshape_as + - resize_ + - resize_as_ + - roll + - rot90 + - round + - round_ + - rsqrt + - rsqrt_ + - scatter + - scatter_ + - scatter_add + - scatter_add_ + - select + - sgn + - sgn_ + - sigmoid + - sigmoid_ + - sign + - sign_ + - signbit + - sin + - sin_ + - sinc + - sinc_ + - sinh + - sinh_ + - slogdet + - smm + - softmax + - solve + - sort + - split_with_sizes + - sqrt + - sqrt_ + - square + - square_ + - squeeze + - squeeze_ + - sspaddmm + - std + - sub + - sub_ + - sum + - sum_to_size + - svd + - symeig + - t + - t_ + - take + - tan + - tan_ + - tanh + - tanh_ + - tensor_split + - tile + - topk + - transpose + - transpose_ + - triangular_solve + - tril + - tril_ + - triu + - true_divide + - triu_ + - true_divide_ + - trunc + - trunc_ + - type_as + - unbind + - unflatten + - unfold + - unsafe_chunk + - unsqueeze + - unsafe_split + - unsafe_split_with_sizes + - var + - vdot + - unsqueeze_ + - view_as + - xlogy + - xlogy_ + +torch: + - _adaptive_avg_pool2d + - _add_relu + - _add_relu_ + - _aminmax + - _batch_norm_impl_index + - _convolution + - abs + - abs_ + - absolute + - acos + - acos_ + - acosh + - acosh_ + - adaptive_avg_pool1d + - adaptive_max_pool1d + - add + - addbmm + - addcdiv + - addcmul + - addmm + - addmv + - addmv_ + - addr + - amax + - affine_grid_generator + - align_tensors + - all + - alpha_dropout + - amin + - alpha_dropout_ + - angle + - any + - arange + - arccos + - arccos_ + - arccosh + - arccosh_ + - arcsin + - arcsin_ + - arcsinh + - arcsinh_ + - arctan + - arctan_ + - arctanh + - arctanh_ + - argmax + - argmin + - argsort + - asin + - asin_ + - asinh + - asinh_ + - atan + - atan2 + - atan_ + - atanh + - atanh_ + - atleast_1d + - atleast_2d + - atleast_3d + - avg_pool1d + - baddbmm + - bartlett_window + - batch_norm_backward_elemt + - batch_norm_backward_reduce + - batch_norm_elemt + - batch_norm_gather_stats + - batch_norm_gather_stats_with_counts + - bernoulli + - batch_norm_stats + - batch_norm_update_stats + - bilinear + - bincount + - binomial + - binary_cross_entropy_with_logits + - bitwise_and + - bitwise_not + - bitwise_or + - bitwise_xor + - blackman_window + - block_diag + - bmm + - broadcast_tensors + - broadcast_to + - cartesian_prod + - cat + - cdist + - ceil + - ceil_ + - celu + - celu_ + - chain_matmul + - channel_shuffle + - cholesky + - cholesky_inverse + - cholesky_solve + - choose_qparams_optimized + - chunk + - clamp + - clamp_ + - clamp_max + - clamp_max_ + - clamp_min + - clamp_min_ + - clip + - clip_ + - clone + - column_stack + - combinations + - constant_pad_nd + - conv1d + - conv2d + - conv3d + - conv_tbc + - conv_transpose1d + - conv_transpose2d + - conv_transpose3d + - cos + - convolution + - copysign + - cos_ + - cosh + - cosh_ + - cosine_embedding_loss + - cosine_similarity + - count_nonzero + - cross + - ctc_loss + - cummax + - cummin + - cumprod + - cumsum + - deg2rad + - deg2rad_ + - det + - diag + - diag_embed + - diff + - diagflat + - diagonal + - digamma + - dist + - div + - divide + - dot + - dropout + - dropout_ + - dsmm + - dstack + - eig + - einsum + - embedding + - embedding_bag + - embedding_renorm_ + - eq + - equal + - erf + - erf_ + - erfc + - erfc_ + - erfinv + - exp + - exp2 + - exp2_ + - exp_ + - expm1 + - expm1_ + - eye + - feature_dropout + - feature_alpha_dropout + - feature_alpha_dropout_ + - feature_dropout_ + - fix + - fill_ + - fix_ + - flatten + - flip + - fliplr + - flipud + - float_power + - floor + - floor_ + - floor_divide + - fmax + - fmin + - fmod + - frac + - frac_ + - full + - frobenius_norm + - full_like + - gather + - gcd + - gcd_ + - ge + - geqrf + - ger + - greater + - greater_equal + - grid_sampler + - grid_sampler_2d + - group_norm + - grid_sampler_3d + - gru + - gru_cell + - gt + - hamming_window + - hann_window + - hardshrink + - heaviside + - hinge_embedding_loss + - histc + - hsmm + - hspmm + - hstack + - hypot + - igamma + - igammac + - index_add + - index_copy + - inner + - index_fill + - index_put + - index_put_ + - index_select + - instance_norm + - isclose + - isfinite + - isinf + - isnan + - isneginf + - isposinf + - istft + - kaiser_window + - kl_div + - kron + - kthvalue + - layer_norm + - lcm + - lcm_ + - ldexp + - ldexp_ + - le + - lerp + - less + - less_equal + - lgamma + - linspace + - log + - log10 + - log10_ + - log1p + - log1p_ + - log2 + - log2_ + - log_softmax + - log_ + - logaddexp + - logaddexp2 + - logcumsumexp + - logdet + - logical_and + - logical_not + - logical_or + - logical_xor + - logit + - logit_ + - logspace + - logsumexp + - lstm + - lstm_cell + - lstsq + - lt + - lu_solve + - masked_fill + - margin_ranking_loss + - masked_scatter + - masked_select + - matrix_exp + - matmul + - matrix_power + - matrix_rank + - max + - max_pool1d + - max_pool2d + - max_pool1d_with_indices + - max_pool3d + - maximum + - mean + - median + - min + - minimum + - mm + - mode + - moveaxis + - movedim + - msort + - mul + - multinomial + - multiply + - mv + - mvlgamma + - nan_to_num + - nan_to_num_ + - nanmedian + - nansum + - narrow + - native_batch_norm + - native_group_norm + - narrow_copy + - native_layer_norm + - native_norm + - ne + - neg + - negative + - neg_ + - negative_ + - nextafter + - nonzero + - norm_except_dim + - normal + - not_equal + - nuclear_norm + - pairwise_distance + - pdist + - pinverse + - pixel_shuffle + - pixel_unshuffle + - poisson + - poisson_nll_loss + - polar + - polygamma + - pow + - prelu + - prod + - rad2deg + - promote_types + - rad2deg_ + - range + - ravel + - real + - reciprocal + - relu + - reciprocal_ + - relu_ + - remainder + - renorm + - repeat_interleave + - reshape + - resize_as_ + - roll + - rot90 + - round + - round_ + - rrelu + - rrelu_ + - rsqrt + - row_stack + - rsqrt_ + - rsub + - saddmm + - scalar_tensor + - scatter + - select + - scatter_add + - searchsorted + - selu + - selu_ + - sgn + - sigmoid + - sigmoid_ + - sign + - signbit + - sin + - sin_ + - sinc + - sinc_ + - sinh + - sinh_ + - slogdet + - smm + - softmax + - solve + - sort + - sparse_coo_tensor + - square + - split_with_sizes + - spmm + - sqrt + - sqrt_ + - square_ + - squeeze + - sspaddmm + - stack + - std + - std_mean + - sub + - subtract + - sum + - svd + - swapaxes + - swapdims + - symeig + - t + - take + - tan + - tan_ + - tanh + - tanh_ + - tensordot + - tensor_split + - threshold + - threshold_ + - tile + - topk + - transpose + - trapz + - triangular_solve + - tril + - tril_indices + - triplet_margin_loss + - triu + - triu_indices + - true_divide + - trunc + - trunc_ + - unique_consecutive + - xlogy + - unbind + - unique_dim + - unsafe_chunk + - unsafe_split + - vander + - var + - vdot + - unsafe_split_with_sizes + - unsqueeze + - var_mean + - vstack + - where + - xlogy_ -- Gitee From 17a1eb1552d49a76ead3d3aa5f12c10f55ccd4bc Mon Sep 17 00:00:00 2001 From: wangchao Date: Tue, 18 Jul 2023 12:26:24 +0000 Subject: [PATCH 09/19] add debug/tools/api_ut_tools/hook_module/wrap_functional.py. Signed-off-by: wangchao --- .../hook_module/wrap_functional.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 debug/tools/api_ut_tools/hook_module/wrap_functional.py diff --git a/debug/tools/api_ut_tools/hook_module/wrap_functional.py b/debug/tools/api_ut_tools/hook_module/wrap_functional.py new file mode 100644 index 00000000000..0eff94c3413 --- /dev/null +++ b/debug/tools/api_ut_tools/hook_module/wrap_functional.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +import os + +import torch +import yaml + +from .hook_module import HOOKModule +from ..common.utils import torch_device_guard + +cur_path = os.path.dirname(os.path.realpath(__file__)) +yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") +with open(yaml_path, 'r') as f: + WrapFunctionalOps = yaml.safe_load(f).get('functional') + +for f in dir(torch.nn.functional): + locals().update({f: getattr(torch.nn.functional, f)}) + + +def get_functional_ops(): + global WrapFunctionalOps + _all_functional_ops = dir(torch.nn.functional) + return set(WrapFunctionalOps) & set(_all_functional_ops) + + +class HOOKFunctionalOP(object): + pass + + +class FunctionalOPTemplate(HOOKModule): + def __init__(self, op_name, hook): + self.op_name_ = op_name + self.prefix_op_name_ = "Functional*" + str(op_name) + "*" + super().__init__(hook) + + @torch_device_guard + def forward(self, *args, **kwargs): + return eval(self.op_name_)(*args, **kwargs) + + +def wrap_functional_op(op_name, hook): + def functional_op_template(*args, **kwargs): + return FunctionalOPTemplate(op_name, hook)(*args, **kwargs) + + return functional_op_template + + +def wrap_functional_ops_and_bind(hook): + _functional_ops = get_functional_ops() + for op_name in _functional_ops: + setattr(HOOKFunctionalOP, "wrap_" + op_name, wrap_functional_op(op_name, hook)) -- Gitee From d80e211ee3121d885e04e66d5c3e910dd5498303 Mon Sep 17 00:00:00 2001 From: wangchao Date: Tue, 18 Jul 2023 12:26:59 +0000 Subject: [PATCH 10/19] add debug/tools/api_ut_tools/hook_module/wrap_tensor.py. Signed-off-by: wangchao --- .../api_ut_tools/hook_module/wrap_tensor.py | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 debug/tools/api_ut_tools/hook_module/wrap_tensor.py diff --git a/debug/tools/api_ut_tools/hook_module/wrap_tensor.py b/debug/tools/api_ut_tools/hook_module/wrap_tensor.py new file mode 100644 index 00000000000..07fbca8711d --- /dev/null +++ b/debug/tools/api_ut_tools/hook_module/wrap_tensor.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +import os + +import torch +import yaml + +from .hook_module import HOOKModule +from ..common.utils import torch_device_guard + +cur_path = os.path.dirname(os.path.realpath(__file__)) +yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") +with open(yaml_path, 'r') as f: + WrapTensorOps = yaml.safe_load(f).get('tensor') + + +def get_tensor_ops(): + global WrapTensorOps + _tensor_ops = dir(torch._C._TensorBase) + return set(WrapTensorOps) & set(_tensor_ops) + + +class HOOKTensor(object): + pass + + +class TensorOPTemplate(HOOKModule): + + def __init__(self, op_name, hook): + self.op_name_ = op_name + self.prefix_op_name_ = "Tensor*" + str(op_name) + "*" + super().__init__(hook) + + @torch_device_guard + def forward(self, *args, **kwargs): + return getattr(torch._C._TensorBase, str(self.op_name_))(*args, **kwargs) + + +def wrap_tensor_op(op_name, hook): + + def tensor_op_template(*args, **kwargs): + return TensorOPTemplate(op_name, hook)(*args, **kwargs) + + return tensor_op_template + + +def wrap_tensor_ops_and_bind(hook): + _tensor_ops = get_tensor_ops() + for op_name in _tensor_ops: + setattr(HOOKTensor, "wrap_" + str(op_name), wrap_tensor_op(op_name, hook)) -- Gitee From 52bcc6f2d22268f356f0e93d2f9d0e066b61be7d Mon Sep 17 00:00:00 2001 From: wangchao Date: Tue, 18 Jul 2023 12:27:32 +0000 Subject: [PATCH 11/19] add debug/tools/api_ut_tools/hook_module/wrap_torch.py. Signed-off-by: wangchao --- .../api_ut_tools/hook_module/wrap_torch.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 debug/tools/api_ut_tools/hook_module/wrap_torch.py diff --git a/debug/tools/api_ut_tools/hook_module/wrap_torch.py b/debug/tools/api_ut_tools/hook_module/wrap_torch.py new file mode 100644 index 00000000000..23334be233f --- /dev/null +++ b/debug/tools/api_ut_tools/hook_module/wrap_torch.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +import os + +import torch +import yaml + +from .hook_module import HOOKModule +from ..common.utils import torch_device_guard + +cur_path = os.path.dirname(os.path.realpath(__file__)) +yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") +with open(yaml_path, 'r') as f: + WrapTorchOps = yaml.safe_load(f).get('torch') + + +def get_torch_ops(): + global WrapTorchOps + _torch_ops = dir(torch._C._VariableFunctionsClass) + return set(WrapTorchOps) & set(_torch_ops) + + +class HOOKTorchOP(object): + pass + + +class TorchOPTemplate(HOOKModule): + + def __init__(self, op_name, hook): + self.op_name_ = op_name + self.prefix_op_name_ = "Torch_" + str(op_name) + "_" + super().__init__(hook) + + def input_param_need_adapt(self): + special_op_list = ["broadcast_tensors"] + for item in special_op_list: + if item in self.op_name_: + return True + return False + + def einsum_adapt(self, *args): + if len(args) < 2: + raise ValueError('einsum(): must specify the equation string and at least one operand, ' + 'or at least one operand and its subscripts list') + equation = None + operands = None + if isinstance(args[0], torch.Tensor): + def parse_subscript(n: int) -> str: + if n == Ellipsis: + return '...' + if n >= 0 and n < 26: + return chr(ord('A') + n) + if n >= 26 and n < 52: + return chr(ord('a') + n - 26) + raise ValueError('einsum(): subscript in subscript list is not within the valid range [0, 52]') + equation = ','.join(''.join(parse_subscript(s) for s in l) for l in args[1::2]) + + if len(args) % 2 == 1: + equation += '->' + ''.join(parse_subscript(s) for s in args[-1]) + operands = args[:-1:2] + else: + operands = args[::2] + else: + equation = args[0] + operands = args[1:] + + if len(operands) == 1 and isinstance(operands[0], (list, tuple)): + _operands = operands[0] + return self.einsum_adapt(equation, *_operands) + return equation, operands + + @torch_device_guard + def forward(self, *args, **kwargs): + if self.input_param_need_adapt(): + return getattr(torch._C._VariableFunctionsClass, str(self.op_name_))(args, **kwargs) + else: + if self.op_name_ == 'einsum': + args = self.einsum_adapt(*args) + return getattr(torch._C._VariableFunctionsClass, str(self.op_name_))(*args, **kwargs) + + +def wrap_torch_op(op_name, hook): + + def torch_op_template(*args, **kwargs): + return TorchOPTemplate(op_name, hook)(*args, **kwargs) + + return torch_op_template + + +def wrap_torch_ops_and_bind(hook): + _torch_ops = get_torch_ops() + for op_name in _torch_ops: + setattr(HOOKTorchOP, "wrap_" + op_name, wrap_torch_op(op_name, hook)) -- Gitee From bc01525f88587e14e04e3a9fda713f2ebe73e479 Mon Sep 17 00:00:00 2001 From: wangchao Date: Tue, 18 Jul 2023 12:28:07 +0000 Subject: [PATCH 12/19] add debug/tools/api_ut_tools/common/utils.py. Signed-off-by: wangchao --- debug/tools/api_ut_tools/common/utils.py | 458 +++++++++++++++++++++++ 1 file changed, 458 insertions(+) create mode 100644 debug/tools/api_ut_tools/common/utils.py diff --git a/debug/tools/api_ut_tools/common/utils.py b/debug/tools/api_ut_tools/common/utils.py new file mode 100644 index 00000000000..39794456099 --- /dev/null +++ b/debug/tools/api_ut_tools/common/utils.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +import collections +import os +import random +import re +import stat +import subprocess +import sys +import time +from datetime import datetime, timezone + +import numpy as np +import torch + +try: + import torch_npu +except ImportError: + is_gpu = True +else: + is_gpu = False + +if not is_gpu: + from torch_npu.utils.device_guard import torch_device_guard as torch_npu_device_guard + +device = collections.namedtuple('device', ['type', 'index']) + + +class Const: + """ + Class for const + """ + MODEL_TYPE = ['.onnx', '.pb', '.om'] + DIM_PATTERN = r"^(-?[0-9]+)(,-?[0-9]+)*" + SEMICOLON = ";" + COLON = ":" + EQUAL = "=" + COMMA = "," + DOT = "." + DUMP_RATIO_MAX = 100 + SUMMERY_DATA_NUMS = 256 + ONE_HUNDRED_MB = 100*1024*1024 + FLOAT_EPSILON = np.finfo(float).eps + SUPPORT_DUMP_MODE = ['api', 'acl'] + ON = 'ON' + OFF = 'OFF' + BACKWARD = 'backward' + FORWARD = 'forward' + + # dump mode + ALL = "all" + LIST = "list" + RANGE = "range" + STACK = "stack" + ACL = "acl" + API_LIST = "api_list" + API_STACK = "api_stack" + DUMP_MODE = [ALL, LIST, RANGE, STACK, ACL, API_LIST, API_STACK] + + API_PATTERN = r"^[A-Za-z0-9]+[_]+([A-Za-z0-9]+[_]*[A-Za-z0-9]+)[_]+[0-9]+[_]+[A-Za-z0-9]+" + WRITE_FLAGS = os.O_WRONLY | os.O_CREAT + WRITE_MODES = stat.S_IWUSR | stat.S_IRUSR + + +class CompareConst: + """ + Class for compare module const + """ + # compare result column name + NPU_NAME = "NPU Name" + BENCH_NAME = "Bench Name" + NPU_DTYPE = "NPU Tensor Dtype" + BENCH_DTYPE = "Bench Tensor Dtype" + NPU_SHAPE = "NPU Tensor Shape" + BENCH_SHAPE = "Bench Tensor Shape" + NPU_MAX = "NPU max" + NPU_MIN = "NPU min" + NPU_MEAN = "NPU mean" + BENCH_MAX = "Bench max" + BENCH_MIN = "Bench min" + BENCH_MEAN = "Bench mean" + COSINE = "Cosine" + MAX_ABS_ERR = "MaxAbsErr" + ACCURACY = "Accuracy Reached or Not" + STACK = "NPU_Stack_Info" + ERROR_MESSAGE = "Err_message" + + # compare result data + NAN = 'Nan' + SHAPE_UNMATCH = 'shape unmatched' + DTYPE_UNMATCH = 'dtype unmatched' + + # accuracy standards + COS_THRESHOLD = 0.99 + MAX_ABS_ERR_THRESHOLD = 0.001 + COS_MAX_THRESHOLD = 0.9 + MAX_ABS_ERR_MAX_THRESHOLD = 1 + ACCURACY_CHECK_YES = "Yes" + ACCURACY_CHECK_NO = "No" + ACCURACY_CHECK_UNMATCH = "Unmatched" + + # error message + NO_BENCH = "No bench data matched." + + +class VersionCheck: + """ + Class for TorchVersion + """ + V1_8 = "1.8" + V1_11 = "1.11" + + @staticmethod + def check_torch_version(version): + torch_version = torch.__version__ + if torch_version.startswith(version): + return True + else: + return False + + +class CompareException(Exception): + """ + Class for Accuracy Compare Exception + """ + NONE_ERROR = 0 + INVALID_PATH_ERROR = 1 + OPEN_FILE_ERROR = 2 + CLOSE_FILE_ERROR = 3 + READ_FILE_ERROR = 4 + WRITE_FILE_ERROR = 5 + INVALID_FILE_ERROR = 6 + PERMISSION_ERROR = 7 + INDEX_OUT_OF_BOUNDS_ERROR = 8 + NO_DUMP_FILE_ERROR = 9 + INVALID_DATA_ERROR = 10 + INVALID_PARAM_ERROR = 11 + INVALID_DUMP_RATIO = 12 + INVALID_DUMP_FILE = 13 + UNKNOWN_ERROR = 14 + INVALID_DUMP_MODE = 15 + PARSE_FILE_ERROR = 16 + INVALID_COMPARE_MODE = 17 + + def __init__(self, code, error_info: str = ""): + super(CompareException, self).__init__() + self.code = code + self.error_info = error_info + + def __str__(self): + return self.error_info + +class DumpException(CompareException): + pass + +def _print_log(level, msg): + current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))) + pid = os.getgid() + print(current_time + "(" + str(pid) + ")-[" + level + "]" + msg) + sys.stdout.flush() + + +def print_info_log(info_msg): + """ + Function Description: + print info log. + Parameter: + info_msg: the info message. + """ + _print_log("INFO", info_msg) + + +def print_error_log(error_msg): + """ + Function Description: + print error log. + Parameter: + error_msg: the error message. + """ + _print_log("ERROR", error_msg) + + +def print_warn_log(warn_msg): + """ + Function Description: + print warn log. + Parameter: + warn_msg: the warning message. + """ + _print_log("WARNING", warn_msg) + + +def check_mode_valid(mode): + if mode not in Const.DUMP_MODE: + msg = "Current mode '%s' is not supported. Please use the field in %s" % \ + (mode, Const.DUMP_MODE) + raise CompareException(CompareException.INVALID_DUMP_MODE, msg) + + +def check_file_or_directory_path(path, isdir=False): + """ + Function Description: + check whether the path is valid + Parameter: + path: the path to check + isdir: the path is dir or file + Exception Description: + when invalid data throw exception + """ + if isdir: + if not os.path.exists(path): + print_error_log('The path {} is not exist.'.format(path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) + + if not os.path.isdir(path): + print_error_log('The path {} is not a directory.'.format(path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) + + if not os.access(path, os.W_OK): + print_error_log( + 'The path {} does not have permission to write. Please check the path permission'.format(path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) + else: + if not os.path.isfile(path): + print_error_log('{} is an invalid file or non-exist.'.format(path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) + + if not os.access(path, os.R_OK): + print_error_log( + 'The path {} does not have permission to read. Please check the path permission'.format(path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) + +def _check_pkl(pkl_file_handle, file_name): + tensor_line = pkl_file_handle.readline() + if len(tensor_line) == 0: + print_error_log("dump file {} have empty line!".format(file_name)) + raise CompareException(CompareException.INVALID_DUMP_FILE) + pkl_file_handle.seek(0, 0) + + +def check_file_mode(npu_pkl, bench_pkl, stack_mode): + npu_pkl_name = os.path.split(npu_pkl)[-1] + bench_pkl_name = os.path.split(bench_pkl)[-1] + + if not npu_pkl_name.startswith("api_stack") and not bench_pkl_name.startswith("api_stack"): + if stack_mode: + print_error_log("The current file does not contain stack information, please turn off the stack_mode") + raise CompareException(CompareException.INVALID_COMPARE_MODE) + elif npu_pkl_name.startswith("api_stack") and bench_pkl_name.startswith("api_stack"): + if not stack_mode: + print_error_log("The current file contains stack information, please turn on the stack_mode") + raise CompareException(CompareException.INVALID_COMPARE_MODE) + else: + print_error_log("The dump mode of the two files is not same, please check the dump files") + raise CompareException(CompareException.INVALID_COMPARE_MODE) + + +def check_file_size(input_file, max_size): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print_error_log('Failed to open "%s". %s' % (input_file, str(os_error))) + raise CompareException(CompareException.INVALID_FILE_ERROR) + if file_size > max_size: + print_error_log('The size (%d) of %s exceeds (%d) bytes, tools not support.' + % (file_size, input_file, max_size)) + raise CompareException(CompareException.INVALID_FILE_ERROR) + + +def get_dump_data_path(dump_dir): + """ + Function Description: + traverse directories and obtain the absolute path of dump data + Parameter: + dump_dir: dump data directory + Return Value: + dump data path,file is exist or file is not exist + """ + dump_data_path = None + file_is_exist = False + + check_file_or_directory_path(dump_dir, True) + for dir_path, sub_paths, files in os.walk(dump_dir): + if len(files) != 0: + dump_data_path = dir_path + file_is_exist = True + break + dump_data_path = dir_path + return dump_data_path, file_is_exist + + +def get_api_name_from_matcher(name): + api_matcher = re.compile(Const.API_PATTERN) + match = api_matcher.match(name) + return match.group(1) if match else "" + + +def modify_dump_path(dump_path, mode): + if mode == Const.ALL: + return dump_path + file_name = os.path.split(dump_path) + mode_file_name = mode + "_" + file_name[-1] + return os.path.join(file_name[0], mode_file_name) + + +def create_directory(dir_path): + """ + Function Description: + creating a directory with specified permissions + Parameter: + dir_path: directory path + Exception Description: + when invalid data throw exception + """ + if not os.path.exists(dir_path): + try: + os.makedirs(dir_path, mode=0o700) + except OSError as ex: + print_error_log( + 'Failed to create {}.Please check the path permission or disk space .{}'.format(dir_path, str(ex))) + raise CompareException(CompareException.INVALID_PATH_ERROR) + + +def execute_command(cmd): + """ + Function Description: + run the following command + Parameter: + cmd: command + Exception Description: + when invalid command throw exception + """ + print_info_log('Execute command:%s' % cmd) + process = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + while process.poll() is None: + line = process.stdout.readline() + line = line.strip() + if line: + print(line) + if process.returncode != 0: + print_error_log('Failed to execute command:%s' % " ".join(cmd)) + raise CompareException(CompareException.INVALID_DATA_ERROR) + + +def save_numpy_data(file_path, data): + """ + save_numpy_data + """ + if not os.path.exists(os.path.dirname(file_path)): + os.makedirs(os.path.dirname(file_path)) + np.save(file_path, data) + + +def parse_arg_value(values): + """ + parse dynamic arg value of atc cmdline + """ + value_list = [] + for item in values.split(Const.SEMICOLON): + value_list.append(parse_value_by_comma(item)) + return value_list + + +def parse_value_by_comma(value): + """ + parse value by comma, like '1,2,4,8' + """ + value_list = [] + value_str_list = value.split(Const.COMMA) + for value_str in value_str_list: + value_str = value_str.strip() + if value_str.isdigit() or value_str == '-1': + value_list.append(int(value_str)) + else: + print_error_log("please check your input shape.") + raise CompareException(CompareException.INVALID_PARAM_ERROR) + return value_list + + +def get_data_len_by_shape(shape): + data_len = 1 + for item in shape: + if item == -1: + print_error_log("please check your input shape, one dim in shape is -1.") + return -1 + data_len = data_len * item + return data_len + + +def add_time_as_suffix(name): + return '{}_{}.csv'.format(name, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) + + +def get_time(): + return datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S") + + +def format_value(value): + return '{:.6f}'.format(value) + + +def torch_device_guard(func): + if is_gpu: + return func + # Parse args/kwargs matched torch.device objects + + @torch_npu_device_guard + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper + + +def seed_all(seed=1234, mode=False): + random.seed(seed) + os.environ['PYTHONHASHSEED'] = str(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.use_deterministic_algorithms(mode) + if is_gpu: + torch.cuda.manual_seed_all(seed) + torch.cuda.manual_seed(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.enable = False + torch.backends.cudnn.benchmark = False + else: + torch_npu.npu.manual_seed_all(seed) + torch_npu.npu.manual_seed(seed) + + +def get_process_rank(model): + print_info_log("Rank id is not provided. Trying to get the rank id of the model.") + try: + device = next(model.parameters()).device + except StopIteration: + print_warn_log('There is no parameter in the model. Fail to get rank id.') + return 0, False + if device.type == 'cpu': + print_warn_log("Warning: the debugger is unable to get the rank id. " + "This may cause the dumpped data to be corrupted in the " + "case of distributed training. (You may ignore this if you are using only one card.) " + "Transfer the model to npu or gpu before register_hook() to avoid this warning.") + return 0, False + else: + return device.index, True -- Gitee From a68ff24a3c4dd34e842963e67434902d949383c1 Mon Sep 17 00:00:00 2001 From: wangchao Date: Tue, 18 Jul 2023 12:28:34 +0000 Subject: [PATCH 13/19] add debug/tools/api_ut_tools/dump/dump.py. Signed-off-by: wangchao --- debug/tools/api_ut_tools/dump/dump.py | 158 ++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 debug/tools/api_ut_tools/dump/dump.py diff --git a/debug/tools/api_ut_tools/dump/dump.py b/debug/tools/api_ut_tools/dump/dump.py new file mode 100644 index 00000000000..97b325a2de6 --- /dev/null +++ b/debug/tools/api_ut_tools/dump/dump.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +import inspect +import json +import os +import stat +import numpy as np +import torch +import threading + +try: + import torch_npu +except ImportError: + is_gpu = True +else: + is_gpu = False + +from .utils import DumpUtil, _set_dump_switch4api_list, make_dump_data_dir + +from ..common.utils import print_warn_log, Const, print_info_log, modify_dump_path +from ..dump.utils import check_writable + +forward_init_status = False +backward_init_status = False + +backward_threading_id = 0 + + +class DataInfo(object): + def __init__(self, data, save_data, summary_data, dtype, shape): + self.data = data + self.save_data = save_data + self.summary_data = summary_data + self.dtype = dtype + self.shape = shape + + +def get_not_float_tensor_info(data): + summary_data = [] + if data.numel() == 0 or data.dtype == torch.bool: + tensor_max = [] + tensor_min = [] + tensor_mean = [] + elif len(data.shape) == 0: + tensor_max = data.cpu().detach().float().numpy().tolist() + tensor_min = data.cpu().detach().float().numpy().tolist() + tensor_mean = data.cpu().detach().float().numpy().tolist() + else: + tensor_max = torch._C._VariableFunctionsClass.max(data).cpu().detach().float().numpy().tolist() + tensor_min = torch._C._VariableFunctionsClass.min(data).cpu().detach().float().numpy().tolist() + tensor_mean = torch._C._VariableFunctionsClass.mean(data.float()).cpu().detach().float().numpy().tolist() + saved_tensor = data.contiguous().cpu().detach().numpy() + summary_data.extend([tensor_max, tensor_min, tensor_mean]) + return DataInfo(data, saved_tensor, summary_data, str(data.dtype), tuple(data.shape)) + + +def get_scalar_data_info(data): + summary_data = [data, data, data] + return DataInfo(data, data, summary_data, str(type(data)), str([])) + + +def get_float_tensor_info(data): + summary_data = [] + tensor_max = torch._C._VariableFunctionsClass.max(data).cpu().detach().float().numpy().tolist() + tensor_min = torch._C._VariableFunctionsClass.min(data).cpu().detach().float().numpy().tolist() + tensor_mean = torch._C._VariableFunctionsClass.mean(data).cpu().detach().float().numpy().tolist() + saved_tensor = data.contiguous().cpu().detach().numpy() + summary_data.extend([tensor_max, tensor_min, tensor_mean]) + return DataInfo(data, saved_tensor, summary_data, str(data.dtype), tuple(data.shape)) + + +def dump_tensor(args): + global data_info + args_list = [] + for x in args: + if isinstance(x, torch.Tensor): + if x.numel() == 0 or len(x.shape) == 0 or not x.is_floating_point(): + data_info = get_not_float_tensor_info(x) + else: + data_info = get_float_tensor_info(x) + arg = {"dtype": data_info.dtype, + "shape": data_info.shape, + "type": "torch.Tensor", + "Max": data_info.summary_data[0], + "Min": data_info.summary_data[1]} + else: + arg = {"value": None, + "type": type(x)} + args_list.append(arg) + return args_list + + +def dump_api_tensor(module, name_template, out_feat, dump_file): + api_params_dict = dict() + api_dict = dict() + if Const.BACKWARD in name_template and DumpUtil.dump_mode != Const.FORWARD: + path = os.path.dirname(dump_file) + dump_file = os.path.join(path, "dump_backward.pkl") + api_params_dict["args"] = dump_tensor(out_feat) + elif Const.BACKWARD not in name_template and DumpUtil.dump_mode != Const.BACKWARD: + if module.input_args: + args_list = dump_tensor(module.input_args) + api_params_dict["args"] = args_list + if module.input_kwargs: + api_params_dict["kwargs"] = module.input_kwargs + api_dict[name_template] = api_params_dict + with os.fdopen(os.open(dump_file, os.O_RDWR | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), + "a") as f: + json.dump(api_dict, f) + f.write('\n') + + +def dump_acc_cmp(name, out_feat, module): + dump_file = DumpUtil.get_dump_path() + _set_dump_switch4api_list(name) + + dump_file = modify_dump_path(dump_file, DumpUtil.dump_switch_mode) + + if DumpUtil.get_dump_switch(): + if DumpUtil.dump_init_enable: + DumpUtil.dump_init_enable = False + DumpUtil.dump_data_dir = make_dump_data_dir(dump_file) \ + if DumpUtil.dump_switch_mode not in [Const.STACK, Const.ACL] else "" + if os.path.exists(dump_file) and not os.path.isdir(dump_file): + check_writable(dump_file) + os.remove(dump_file) + + if DumpUtil.dump_switch_mode in [Const.ALL, Const.API_LIST]: + dump_api_tensor(module, name, out_feat, dump_file) + + + + +def acc_cmp_dump(name): + + def acc_cmp_hook(module, in_feat, out_feat): + dump_acc_cmp(name, out_feat, module) + if hasattr(module, "input_args"): + del module.input_args + if hasattr(module, "input_kwargs"): + del module.input_kwargs + + return acc_cmp_hook -- Gitee From afb05d60ec89331735f42a7eee05d2e805f82210 Mon Sep 17 00:00:00 2001 From: wangchao Date: Tue, 18 Jul 2023 12:28:54 +0000 Subject: [PATCH 14/19] add debug/tools/api_ut_tools/dump/utils.py. Signed-off-by: wangchao --- debug/tools/api_ut_tools/dump/utils.py | 219 +++++++++++++++++++++++++ 1 file changed, 219 insertions(+) create mode 100644 debug/tools/api_ut_tools/dump/utils.py diff --git a/debug/tools/api_ut_tools/dump/utils.py b/debug/tools/api_ut_tools/dump/utils.py new file mode 100644 index 00000000000..bebb693945a --- /dev/null +++ b/debug/tools/api_ut_tools/dump/utils.py @@ -0,0 +1,219 @@ +import os +import shutil +import sys +from pathlib import Path + +from ..common.utils import print_error_log, CompareException, DumpException, Const, get_time, print_info_log, \ + check_mode_valid, get_api_name_from_matcher + +from ..common.version import __version__ + +dump_count = 0 +range_begin_flag, range_end_flag = False, False + + +class DumpUtil(object): + dump_data_dir = None + dump_path = None + dump_switch = None + dump_switch_mode = Const.ALL + dump_switch_scope = [] + dump_init_enable = False + dump_api_list = [] + dump_filter_switch = None + dump_mode = Const.ALL + backward_input = {} + dump_dir_tag = 'ptdbg_dump' + dump_config = None + + @staticmethod + def set_dump_path(save_path): + DumpUtil.dump_path = save_path + DumpUtil.dump_init_enable = True + + @staticmethod + def set_dump_config(dump_config): + DumpUtil.dump_config = dump_config + + @staticmethod + def set_dump_switch(switch, mode, scope, api_list, filter_switch, dump_mode): + DumpUtil.dump_switch = switch + DumpUtil.dump_switch_mode = mode + DumpUtil.dump_init_enable = True + DumpUtil.dump_switch_scope = scope + DumpUtil.dump_api_list = [api.lower() for api in api_list] + DumpUtil.dump_filter_switch = filter_switch + DumpUtil.dump_mode = dump_mode + if mode == Const.ACL: + DumpUtil.dump_switch_scope = [api_name.replace("backward", "forward") for api_name in scope] + + def check_list_or_acl_mode(name_prefix): + global dump_count + for item in DumpUtil.dump_switch_scope: + if name_prefix.startswith(item): + dump_count = dump_count + 1 + return True + + def check_range_mode(name_prefix): + global range_begin_flag + global range_end_flag + if name_prefix.startswith(DumpUtil.dump_switch_scope[0]): + range_begin_flag = True + return True + if name_prefix.startswith(DumpUtil.dump_switch_scope[1]): + range_end_flag = True + return True + if range_begin_flag and not range_end_flag: + return True + return False + + def check_stack_mode(name_prefix): + if len(DumpUtil.dump_switch_scope) == 0: + return True + elif len(DumpUtil.dump_switch_scope) == 1: + return name_prefix.startswith(DumpUtil.dump_switch_scope[0]) + elif len(DumpUtil.dump_switch_scope) == 2: + return DumpUtil.check_range_mode(name_prefix) + else: + print_error_log("dump scope is invalid, Please set the scope mode in" + " set_dump_switch with 'all', 'list', 'range', 'stack', 'acl', 'api_list'!") + return False + + check_mapper = { + Const.LIST: check_list_or_acl_mode, + Const.ACL: check_list_or_acl_mode, + Const.RANGE: check_range_mode, + Const.STACK: check_stack_mode + } + + @staticmethod + def check_switch_scope(name_prefix): + if DumpUtil.dump_switch_mode in DumpUtil.check_mapper: + check_func = DumpUtil.check_mapper[DumpUtil.dump_switch_mode] + return check_func(name_prefix) + return False + + @staticmethod + def get_dump_path(): + if DumpUtil.dump_path: + return DumpUtil.dump_path + + if DumpUtil.dump_switch_mode == Const.ALL: + raise RuntimeError("get_dump_path: the file path is empty," + " you must use set_dump_path to set a valid dump path!!!") + else: + dir_path = os.path.realpath("./") + dump_file_name = "scope_dump_{}_{}_{}.pkl".format( + DumpUtil.dump_switch_mode, DumpUtil.dump_switch_scope[0], get_time()) + DumpUtil.dump_path = os.path.join(dir_path, dump_file_name) + return DumpUtil.dump_path + + @staticmethod + def get_dump_switch(): + return DumpUtil.dump_switch == "ON" + + +def set_dump_path(fpath=None, dump_tag='ptdbg_dump'): + if fpath is None: + raise RuntimeError("set_dump_path '{}' error, please set a valid filename".format(fpath)) + return + real_path = os.path.realpath(fpath) + if not os.path.isdir(real_path): + print_error_log( + "set_dump_path '{}' error, the path is not a directory please set a valid directory.".format(real_path)) + raise DumpException(DumpException.INVALID_PATH_ERROR) + DumpUtil.set_dump_path(real_path) + DumpUtil.dump_dir_tag = dump_tag + + +def generate_dump_path_str(): + if DumpUtil.dump_switch_mode == 'acl': + if DumpUtil.dump_config == '': + print_error_log("Please provide dump config for register hook before turning on dump switch!") + raise DumpException(DumpException.NONE_ERROR) + dump_path = f"according to dump config {DumpUtil.dump_config}" + else: + dump_path = f"to {DumpUtil.dump_path}" + return dump_path + + +def set_dump_switch(switch, mode=Const.ALL, scope=[], api_list=[], filter_switch=Const.ON, dump_mode=Const.ALL): + try: + check_mode_valid(mode) + assert switch in ["ON", "OFF"], "Please set dump switch with 'ON' or 'OFF'." + assert filter_switch in ["ON", "OFF"], "Please set filter_switch with 'ON' or 'OFF'." + assert dump_mode in ["all", "forward", "backward"], "Please set dump_mode with 'all' or 'forward' or 'backward'." + if mode == Const.RANGE: + assert len(scope) == 2, "set_dump_switch, scope param set invalid, it's must be [start, end]." + if mode == Const.LIST: + assert len(scope) != 0, "set_dump_switch, scope param set invalid, it's should not be an empty list." + if mode == Const.STACK: + assert len(scope) <= 2, "set_dump_switch, scope param set invalid, it's must be [start, end] or []." + if mode == Const.ACL: + assert len(scope) == 1, "set_dump_switch, scope param set invalid, only one api name is supported in acl mode." + if mode == Const.API_LIST: + assert isinstance(api_list, list) and len(api_list) >= 1, \ + "Current dump mode is 'api_list', but the content of api_list parameter is empty or valid." + except (CompareException, AssertionError) as err: + print_error_log(str(err)) + sys.exit() + + if switch == "OFF": + dump_path_str = generate_dump_path_str() + DumpUtil.set_dump_switch(switch, mode=mode, scope=scope, api_list=api_list, filter_switch=filter_switch, dump_mode=dump_mode) + if switch == "ON": + dump_path_str = generate_dump_path_str() + + global dump_count + if switch == "ON": + print_info_log(f"Dump switch is turned on. Dump data will be saved {dump_path_str}. ") + if mode == Const.LIST: + dump_count = 0 + else: + print_info_log(f"Dump switch is turned off. Dump data has been saved {dump_path_str}. ") + if mode == Const.LIST: + print_info_log("The number of matched dump is {}".format(dump_count)) + +def _set_dump_switch4api_list(name): + if DumpUtil.dump_api_list: + api_name = get_api_name_from_matcher(name) + DumpUtil.dump_switch = "ON" if api_name in DumpUtil.dump_api_list else "OFF" + + +def set_backward_input(backward_input): + for index, api_name in enumerate(DumpUtil.dump_switch_scope): + DumpUtil.backward_input[api_name] = backward_input[index] + + +def make_dump_data_dir(dump_file_name): + dump_path, file_name = os.path.split(os.path.realpath(dump_file_name)) + name_body, name_extension = os.path.splitext(file_name) + output_dir = os.path.join(dump_path, f"{name_body}") + if not os.path.exists(output_dir): + os.mkdir(output_dir, mode=0o750) + else: + shutil.rmtree(output_dir, ignore_errors=True) + os.mkdir(output_dir, mode=0o750) + return output_dir + + +def make_dump_dirs(rank): + dump_file_name, dump_file_name_body = "dump.pkl", "dump" + dump_root_dir = DumpUtil.dump_path if DumpUtil.dump_path else "./" + tag_dir = os.path.join(dump_root_dir, DumpUtil.dump_dir_tag + f'_v{__version__}') + Path(tag_dir).mkdir(mode=0o750, parents=True, exist_ok=True) + rank_dir = os.path.join(tag_dir, 'rank' + str(rank)) + if not os.path.exists(rank_dir): + os.mkdir(rank_dir, mode=0o750) + DumpUtil.dump_dir = rank_dir + dump_file_path = os.path.join(rank_dir, dump_file_name) + DumpUtil.set_dump_path(dump_file_path) + + +def check_writable(dump_file): + if not os.access(dump_file, os.W_OK): + print_error_log( + 'The path {} does not have permission to write. Please check the path permission'.format( + dump_file)) + raise DumpException(DumpException.INVALID_PATH_ERROR) + -- Gitee From dcf513a33e3a6f1dab62910261d0a73a18a36d5f Mon Sep 17 00:00:00 2001 From: wangchao Date: Tue, 18 Jul 2023 12:29:53 +0000 Subject: [PATCH 15/19] add debug/tools/api_ut_tools/run_ut/run_ut.py. Signed-off-by: wangchao --- debug/tools/api_ut_tools/run_ut/run_ut.py | 91 +++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 debug/tools/api_ut_tools/run_ut/run_ut.py diff --git a/debug/tools/api_ut_tools/run_ut/run_ut.py b/debug/tools/api_ut_tools/run_ut/run_ut.py new file mode 100644 index 00000000000..5f753e57af7 --- /dev/null +++ b/debug/tools/api_ut_tools/run_ut/run_ut.py @@ -0,0 +1,91 @@ +import yaml +import os +import json +import torch + +FLOAT_TYPE = ['torch.float32', 'torch.float', 'torch.float64', 'torch.double', 'torch.float16', \ + 'torch.half', 'torch.bfloat16'] + +cur_path = os.path.dirname(os.path.realpath(__file__)) +yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") +with open(yaml_path, 'r') as f: + WrapFunctionalOps = yaml.safe_load(f).get('functional') + +for f in dir(torch.nn.functional): + locals().update({f: getattr(torch.nn.functional, f)}) + + +def run_ut(): + print("start") + forward_pkl = open("/home/wangchao/torch_test/dump_data_new/npu/ptdbg_dump_v1.0/rank0/dump.pkl") + backward_pkl = open("/home/wangchao/torch_test/dump_data_new/npu/ptdbg_dump_v1.0/rank0/dump_backward.pkl") + forward_content = forward_pkl.readlines() + backward_content = backward_pkl.readlines() + for api_info in forward_content: + api_json = json.loads(api_info) + for key, value in api_json.items(): + [api_type, api_name, index, mode] = key.split("*") + print(api_name) + api_feature = key.rsplit("*", 1)[0] + args, kwargs = generate_input(value.get("args"), api_json.get("kwargs")) + if api_type == "Functional": + out = eval(api_name)(*args, **kwargs) + if api_type == "Tensor": + out = getattr(torch._C._TensorBase, str(api_name))(*args, **kwargs) + for line in backward_content: + if api_feature in line: + api_back_json = json.loads(line) + for params in api_back_json.values(): + grad = nested_generate_input(params.get("args"), True, False) + out.backward(grad) + input_grad = [tensor.grad for tensor in args if isinstance(tensor, torch.Tensor)] + print("forward") + print(out) + print("backward") + print(input_grad) + + +def generate_input(input_args, input_kwargs, need_backward=True, need_convert=False): # 没有考虑dict of tensor + args = [] + kwargs = {} + + for info in input_args: + args.append(nested_generate_input(info, need_backward, need_convert)) + if kwargs: + for key, info in input_kwargs.items(): + kwargs[key] = nested_generate_input(info, need_backward, need_convert) + return args, kwargs + + +def nested_generate_input(info, need_backward, need_convert): + if isinstance(info, list): + result = [] + for i in info: + result.append(nested_generate_input(i, need_backward, need_convert)) + return result + # return list(map(nested_generate_input, info)) + # elif isinstance(input_info, tuple): + # return tuple(map(generate_input, input_info)) + else: + if info['type'] == 'torch.Tensor': + low, high = info['Min'], info['Max'] + data_dtype = info['dtype'] + if data_dtype in FLOAT_TYPE: #应该搞个float类型列表 + if need_convert and data_dtype == "torch.float16": + data_dtype = "torch.float32" + scale = high - low + rand01 = torch.rand(tuple(info['shape']), dtype=eval(data_dtype)) + inpt = rand01 * scale + low + if need_backward: + inpt.requires_grad_(True) + inpt.retain_grad() + elif 'int' in data_dtype or 'long' in data_dtype: # 应该搞个int类型列表, + inpt = torch.randint(int(low), int(high)+1, tuple(info['shape']), dtype=eval(data_dtype)) # high + 1因为右边是开区间 + else: + print(f'Warning: Dtype is not supported: ', info['dtype']) + raise NotImplementedError() + else: + inpt = info['value'] # 遗留问题：需要考虑是否要转换成原本类型 + return inpt + +run_ut() \ No newline at end of file -- Gitee From 34884c4d1eba3dad3313aa0196719b1a17bdd59c Mon Sep 17 00:00:00 2001 From: wangchao Date: Tue, 18 Jul 2023 12:30:22 +0000 Subject: [PATCH 16/19] add debug/tools/api_ut_tools/__init__.py. Signed-off-by: wangchao --- debug/tools/api_ut_tools/__init__.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 debug/tools/api_ut_tools/__init__.py diff --git a/debug/tools/api_ut_tools/__init__.py b/debug/tools/api_ut_tools/__init__.py new file mode 100644 index 00000000000..9b62b9512de --- /dev/null +++ b/debug/tools/api_ut_tools/__init__.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" + +from .dump.dump import acc_cmp_dump +from .dump.utils import set_dump_path, set_dump_switch, set_backward_input +from .hook_module.register_hook import register_hook +from .common.utils import seed_all +from .common.version import __version__ +seed_all() + +__all__ = ["register_hook", "set_dump_path", "set_dump_switch", "seed_all", + "acc_cmp_dump"] -- Gitee From 87bdee05b0740b266e585a3a613575c977aab5f2 Mon Sep 17 00:00:00 2001 From: wangchao Date: Fri, 21 Jul 2023 14:55:37 +0800 Subject: [PATCH 17/19] api ut tools first commit --- precision/api_ut_tools/.keep | 0 precision/api_ut_tools/__init__.py | 27 + precision/api_ut_tools/common/.keep | 0 precision/api_ut_tools/common/utils.py | 458 ++++++++ precision/api_ut_tools/compare/algorithm.py | 1 + precision/api_ut_tools/compare/compare.py | 1 + precision/api_ut_tools/compare/save_result.py | 1 + precision/api_ut_tools/dump/.keep | 0 precision/api_ut_tools/dump/api_info.py | 6 + precision/api_ut_tools/dump/dump.py | 158 +++ precision/api_ut_tools/dump/dump_scope.py | 1 + precision/api_ut_tools/dump/info_dump.py | 1 + precision/api_ut_tools/dump/utils.py | 219 ++++ precision/api_ut_tools/hook_module/.keep | 0 .../api_ut_tools/hook_module/hook_module.py | 113 ++ .../api_ut_tools/hook_module/register_hook.py | 62 + .../hook_module/support_wrap_ops.yaml | 1000 +++++++++++++++++ .../hook_module/wrap_functional.py | 66 ++ .../api_ut_tools/hook_module/wrap_tensor.py | 65 ++ .../api_ut_tools/hook_module/wrap_torch.py | 108 ++ precision/api_ut_tools/run_ut/.keep | 0 .../api_ut_tools/run_ut/data_generate.py | 1 + precision/api_ut_tools/run_ut/run_ut.py | 92 ++ 23 files changed, 2380 insertions(+) create mode 100644 precision/api_ut_tools/.keep create mode 100644 precision/api_ut_tools/__init__.py create mode 100644 precision/api_ut_tools/common/.keep create mode 100644 precision/api_ut_tools/common/utils.py create mode 100644 precision/api_ut_tools/compare/algorithm.py create mode 100644 precision/api_ut_tools/compare/compare.py create mode 100644 precision/api_ut_tools/compare/save_result.py create mode 100644 precision/api_ut_tools/dump/.keep create mode 100644 precision/api_ut_tools/dump/api_info.py create mode 100644 precision/api_ut_tools/dump/dump.py create mode 100644 precision/api_ut_tools/dump/dump_scope.py create mode 100644 precision/api_ut_tools/dump/info_dump.py create mode 100644 precision/api_ut_tools/dump/utils.py create mode 100644 precision/api_ut_tools/hook_module/.keep create mode 100644 precision/api_ut_tools/hook_module/hook_module.py create mode 100644 precision/api_ut_tools/hook_module/register_hook.py create mode 100644 precision/api_ut_tools/hook_module/support_wrap_ops.yaml create mode 100644 precision/api_ut_tools/hook_module/wrap_functional.py create mode 100644 precision/api_ut_tools/hook_module/wrap_tensor.py create mode 100644 precision/api_ut_tools/hook_module/wrap_torch.py create mode 100644 precision/api_ut_tools/run_ut/.keep create mode 100644 precision/api_ut_tools/run_ut/data_generate.py create mode 100644 precision/api_ut_tools/run_ut/run_ut.py diff --git a/precision/api_ut_tools/.keep b/precision/api_ut_tools/.keep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/precision/api_ut_tools/__init__.py b/precision/api_ut_tools/__init__.py new file mode 100644 index 00000000000..9b62b9512de --- /dev/null +++ b/precision/api_ut_tools/__init__.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" + +from .dump.dump import acc_cmp_dump +from .dump.utils import set_dump_path, set_dump_switch, set_backward_input +from .hook_module.register_hook import register_hook +from .common.utils import seed_all +from .common.version import __version__ +seed_all() + +__all__ = ["register_hook", "set_dump_path", "set_dump_switch", "seed_all", + "acc_cmp_dump"] diff --git a/precision/api_ut_tools/common/.keep b/precision/api_ut_tools/common/.keep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/precision/api_ut_tools/common/utils.py b/precision/api_ut_tools/common/utils.py new file mode 100644 index 00000000000..39794456099 --- /dev/null +++ b/precision/api_ut_tools/common/utils.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +import collections +import os +import random +import re +import stat +import subprocess +import sys +import time +from datetime import datetime, timezone + +import numpy as np +import torch + +try: + import torch_npu +except ImportError: + is_gpu = True +else: + is_gpu = False + +if not is_gpu: + from torch_npu.utils.device_guard import torch_device_guard as torch_npu_device_guard + +device = collections.namedtuple('device', ['type', 'index']) + + +class Const: + """ + Class for const + """ + MODEL_TYPE = ['.onnx', '.pb', '.om'] + DIM_PATTERN = r"^(-?[0-9]+)(,-?[0-9]+)*" + SEMICOLON = ";" + COLON = ":" + EQUAL = "=" + COMMA = "," + DOT = "." + DUMP_RATIO_MAX = 100 + SUMMERY_DATA_NUMS = 256 + ONE_HUNDRED_MB = 100*1024*1024 + FLOAT_EPSILON = np.finfo(float).eps + SUPPORT_DUMP_MODE = ['api', 'acl'] + ON = 'ON' + OFF = 'OFF' + BACKWARD = 'backward' + FORWARD = 'forward' + + # dump mode + ALL = "all" + LIST = "list" + RANGE = "range" + STACK = "stack" + ACL = "acl" + API_LIST = "api_list" + API_STACK = "api_stack" + DUMP_MODE = [ALL, LIST, RANGE, STACK, ACL, API_LIST, API_STACK] + + API_PATTERN = r"^[A-Za-z0-9]+[_]+([A-Za-z0-9]+[_]*[A-Za-z0-9]+)[_]+[0-9]+[_]+[A-Za-z0-9]+" + WRITE_FLAGS = os.O_WRONLY | os.O_CREAT + WRITE_MODES = stat.S_IWUSR | stat.S_IRUSR + + +class CompareConst: + """ + Class for compare module const + """ + # compare result column name + NPU_NAME = "NPU Name" + BENCH_NAME = "Bench Name" + NPU_DTYPE = "NPU Tensor Dtype" + BENCH_DTYPE = "Bench Tensor Dtype" + NPU_SHAPE = "NPU Tensor Shape" + BENCH_SHAPE = "Bench Tensor Shape" + NPU_MAX = "NPU max" + NPU_MIN = "NPU min" + NPU_MEAN = "NPU mean" + BENCH_MAX = "Bench max" + BENCH_MIN = "Bench min" + BENCH_MEAN = "Bench mean" + COSINE = "Cosine" + MAX_ABS_ERR = "MaxAbsErr" + ACCURACY = "Accuracy Reached or Not" + STACK = "NPU_Stack_Info" + ERROR_MESSAGE = "Err_message" + + # compare result data + NAN = 'Nan' + SHAPE_UNMATCH = 'shape unmatched' + DTYPE_UNMATCH = 'dtype unmatched' + + # accuracy standards + COS_THRESHOLD = 0.99 + MAX_ABS_ERR_THRESHOLD = 0.001 + COS_MAX_THRESHOLD = 0.9 + MAX_ABS_ERR_MAX_THRESHOLD = 1 + ACCURACY_CHECK_YES = "Yes" + ACCURACY_CHECK_NO = "No" + ACCURACY_CHECK_UNMATCH = "Unmatched" + + # error message + NO_BENCH = "No bench data matched." + + +class VersionCheck: + """ + Class for TorchVersion + """ + V1_8 = "1.8" + V1_11 = "1.11" + + @staticmethod + def check_torch_version(version): + torch_version = torch.__version__ + if torch_version.startswith(version): + return True + else: + return False + + +class CompareException(Exception): + """ + Class for Accuracy Compare Exception + """ + NONE_ERROR = 0 + INVALID_PATH_ERROR = 1 + OPEN_FILE_ERROR = 2 + CLOSE_FILE_ERROR = 3 + READ_FILE_ERROR = 4 + WRITE_FILE_ERROR = 5 + INVALID_FILE_ERROR = 6 + PERMISSION_ERROR = 7 + INDEX_OUT_OF_BOUNDS_ERROR = 8 + NO_DUMP_FILE_ERROR = 9 + INVALID_DATA_ERROR = 10 + INVALID_PARAM_ERROR = 11 + INVALID_DUMP_RATIO = 12 + INVALID_DUMP_FILE = 13 + UNKNOWN_ERROR = 14 + INVALID_DUMP_MODE = 15 + PARSE_FILE_ERROR = 16 + INVALID_COMPARE_MODE = 17 + + def __init__(self, code, error_info: str = ""): + super(CompareException, self).__init__() + self.code = code + self.error_info = error_info + + def __str__(self): + return self.error_info + +class DumpException(CompareException): + pass + +def _print_log(level, msg): + current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))) + pid = os.getgid() + print(current_time + "(" + str(pid) + ")-[" + level + "]" + msg) + sys.stdout.flush() + + +def print_info_log(info_msg): + """ + Function Description: + print info log. + Parameter: + info_msg: the info message. + """ + _print_log("INFO", info_msg) + + +def print_error_log(error_msg): + """ + Function Description: + print error log. + Parameter: + error_msg: the error message. + """ + _print_log("ERROR", error_msg) + + +def print_warn_log(warn_msg): + """ + Function Description: + print warn log. + Parameter: + warn_msg: the warning message. + """ + _print_log("WARNING", warn_msg) + + +def check_mode_valid(mode): + if mode not in Const.DUMP_MODE: + msg = "Current mode '%s' is not supported. Please use the field in %s" % \ + (mode, Const.DUMP_MODE) + raise CompareException(CompareException.INVALID_DUMP_MODE, msg) + + +def check_file_or_directory_path(path, isdir=False): + """ + Function Description: + check whether the path is valid + Parameter: + path: the path to check + isdir: the path is dir or file + Exception Description: + when invalid data throw exception + """ + if isdir: + if not os.path.exists(path): + print_error_log('The path {} is not exist.'.format(path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) + + if not os.path.isdir(path): + print_error_log('The path {} is not a directory.'.format(path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) + + if not os.access(path, os.W_OK): + print_error_log( + 'The path {} does not have permission to write. Please check the path permission'.format(path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) + else: + if not os.path.isfile(path): + print_error_log('{} is an invalid file or non-exist.'.format(path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) + + if not os.access(path, os.R_OK): + print_error_log( + 'The path {} does not have permission to read. Please check the path permission'.format(path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) + +def _check_pkl(pkl_file_handle, file_name): + tensor_line = pkl_file_handle.readline() + if len(tensor_line) == 0: + print_error_log("dump file {} have empty line!".format(file_name)) + raise CompareException(CompareException.INVALID_DUMP_FILE) + pkl_file_handle.seek(0, 0) + + +def check_file_mode(npu_pkl, bench_pkl, stack_mode): + npu_pkl_name = os.path.split(npu_pkl)[-1] + bench_pkl_name = os.path.split(bench_pkl)[-1] + + if not npu_pkl_name.startswith("api_stack") and not bench_pkl_name.startswith("api_stack"): + if stack_mode: + print_error_log("The current file does not contain stack information, please turn off the stack_mode") + raise CompareException(CompareException.INVALID_COMPARE_MODE) + elif npu_pkl_name.startswith("api_stack") and bench_pkl_name.startswith("api_stack"): + if not stack_mode: + print_error_log("The current file contains stack information, please turn on the stack_mode") + raise CompareException(CompareException.INVALID_COMPARE_MODE) + else: + print_error_log("The dump mode of the two files is not same, please check the dump files") + raise CompareException(CompareException.INVALID_COMPARE_MODE) + + +def check_file_size(input_file, max_size): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print_error_log('Failed to open "%s". %s' % (input_file, str(os_error))) + raise CompareException(CompareException.INVALID_FILE_ERROR) + if file_size > max_size: + print_error_log('The size (%d) of %s exceeds (%d) bytes, tools not support.' + % (file_size, input_file, max_size)) + raise CompareException(CompareException.INVALID_FILE_ERROR) + + +def get_dump_data_path(dump_dir): + """ + Function Description: + traverse directories and obtain the absolute path of dump data + Parameter: + dump_dir: dump data directory + Return Value: + dump data path,file is exist or file is not exist + """ + dump_data_path = None + file_is_exist = False + + check_file_or_directory_path(dump_dir, True) + for dir_path, sub_paths, files in os.walk(dump_dir): + if len(files) != 0: + dump_data_path = dir_path + file_is_exist = True + break + dump_data_path = dir_path + return dump_data_path, file_is_exist + + +def get_api_name_from_matcher(name): + api_matcher = re.compile(Const.API_PATTERN) + match = api_matcher.match(name) + return match.group(1) if match else "" + + +def modify_dump_path(dump_path, mode): + if mode == Const.ALL: + return dump_path + file_name = os.path.split(dump_path) + mode_file_name = mode + "_" + file_name[-1] + return os.path.join(file_name[0], mode_file_name) + + +def create_directory(dir_path): + """ + Function Description: + creating a directory with specified permissions + Parameter: + dir_path: directory path + Exception Description: + when invalid data throw exception + """ + if not os.path.exists(dir_path): + try: + os.makedirs(dir_path, mode=0o700) + except OSError as ex: + print_error_log( + 'Failed to create {}.Please check the path permission or disk space .{}'.format(dir_path, str(ex))) + raise CompareException(CompareException.INVALID_PATH_ERROR) + + +def execute_command(cmd): + """ + Function Description: + run the following command + Parameter: + cmd: command + Exception Description: + when invalid command throw exception + """ + print_info_log('Execute command:%s' % cmd) + process = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + while process.poll() is None: + line = process.stdout.readline() + line = line.strip() + if line: + print(line) + if process.returncode != 0: + print_error_log('Failed to execute command:%s' % " ".join(cmd)) + raise CompareException(CompareException.INVALID_DATA_ERROR) + + +def save_numpy_data(file_path, data): + """ + save_numpy_data + """ + if not os.path.exists(os.path.dirname(file_path)): + os.makedirs(os.path.dirname(file_path)) + np.save(file_path, data) + + +def parse_arg_value(values): + """ + parse dynamic arg value of atc cmdline + """ + value_list = [] + for item in values.split(Const.SEMICOLON): + value_list.append(parse_value_by_comma(item)) + return value_list + + +def parse_value_by_comma(value): + """ + parse value by comma, like '1,2,4,8' + """ + value_list = [] + value_str_list = value.split(Const.COMMA) + for value_str in value_str_list: + value_str = value_str.strip() + if value_str.isdigit() or value_str == '-1': + value_list.append(int(value_str)) + else: + print_error_log("please check your input shape.") + raise CompareException(CompareException.INVALID_PARAM_ERROR) + return value_list + + +def get_data_len_by_shape(shape): + data_len = 1 + for item in shape: + if item == -1: + print_error_log("please check your input shape, one dim in shape is -1.") + return -1 + data_len = data_len * item + return data_len + + +def add_time_as_suffix(name): + return '{}_{}.csv'.format(name, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) + + +def get_time(): + return datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S") + + +def format_value(value): + return '{:.6f}'.format(value) + + +def torch_device_guard(func): + if is_gpu: + return func + # Parse args/kwargs matched torch.device objects + + @torch_npu_device_guard + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper + + +def seed_all(seed=1234, mode=False): + random.seed(seed) + os.environ['PYTHONHASHSEED'] = str(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.use_deterministic_algorithms(mode) + if is_gpu: + torch.cuda.manual_seed_all(seed) + torch.cuda.manual_seed(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.enable = False + torch.backends.cudnn.benchmark = False + else: + torch_npu.npu.manual_seed_all(seed) + torch_npu.npu.manual_seed(seed) + + +def get_process_rank(model): + print_info_log("Rank id is not provided. Trying to get the rank id of the model.") + try: + device = next(model.parameters()).device + except StopIteration: + print_warn_log('There is no parameter in the model. Fail to get rank id.') + return 0, False + if device.type == 'cpu': + print_warn_log("Warning: the debugger is unable to get the rank id. " + "This may cause the dumpped data to be corrupted in the " + "case of distributed training. (You may ignore this if you are using only one card.) " + "Transfer the model to npu or gpu before register_hook() to avoid this warning.") + return 0, False + else: + return device.index, True diff --git a/precision/api_ut_tools/compare/algorithm.py b/precision/api_ut_tools/compare/algorithm.py new file mode 100644 index 00000000000..cbb64cd1961 --- /dev/null +++ b/precision/api_ut_tools/compare/algorithm.py @@ -0,0 +1 @@ +# 定义比对算法及比对标准 \ No newline at end of file diff --git a/precision/api_ut_tools/compare/compare.py b/precision/api_ut_tools/compare/compare.py new file mode 100644 index 00000000000..c4c64321701 --- /dev/null +++ b/precision/api_ut_tools/compare/compare.py @@ -0,0 +1 @@ +# 进行比对及结果展示 \ No newline at end of file diff --git a/precision/api_ut_tools/compare/save_result.py b/precision/api_ut_tools/compare/save_result.py new file mode 100644 index 00000000000..06191a4a54f --- /dev/null +++ b/precision/api_ut_tools/compare/save_result.py @@ -0,0 +1 @@ +# 比对表格输出，并将失败api数据进行保存 \ No newline at end of file diff --git a/precision/api_ut_tools/dump/.keep b/precision/api_ut_tools/dump/.keep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/precision/api_ut_tools/dump/api_info.py b/precision/api_ut_tools/dump/api_info.py new file mode 100644 index 00000000000..5c6138f6e48 --- /dev/null +++ b/precision/api_ut_tools/dump/api_info.py @@ -0,0 +1,6 @@ +# 定义API INFO，保存基本信息，用于后续结构体的落盘，注意考虑random场景及真实数据场景 + + +class APIInfo: + def __init__(self, api_name): + self.api_name = api_name diff --git a/precision/api_ut_tools/dump/dump.py b/precision/api_ut_tools/dump/dump.py new file mode 100644 index 00000000000..97b325a2de6 --- /dev/null +++ b/precision/api_ut_tools/dump/dump.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +import inspect +import json +import os +import stat +import numpy as np +import torch +import threading + +try: + import torch_npu +except ImportError: + is_gpu = True +else: + is_gpu = False + +from .utils import DumpUtil, _set_dump_switch4api_list, make_dump_data_dir + +from ..common.utils import print_warn_log, Const, print_info_log, modify_dump_path +from ..dump.utils import check_writable + +forward_init_status = False +backward_init_status = False + +backward_threading_id = 0 + + +class DataInfo(object): + def __init__(self, data, save_data, summary_data, dtype, shape): + self.data = data + self.save_data = save_data + self.summary_data = summary_data + self.dtype = dtype + self.shape = shape + + +def get_not_float_tensor_info(data): + summary_data = [] + if data.numel() == 0 or data.dtype == torch.bool: + tensor_max = [] + tensor_min = [] + tensor_mean = [] + elif len(data.shape) == 0: + tensor_max = data.cpu().detach().float().numpy().tolist() + tensor_min = data.cpu().detach().float().numpy().tolist() + tensor_mean = data.cpu().detach().float().numpy().tolist() + else: + tensor_max = torch._C._VariableFunctionsClass.max(data).cpu().detach().float().numpy().tolist() + tensor_min = torch._C._VariableFunctionsClass.min(data).cpu().detach().float().numpy().tolist() + tensor_mean = torch._C._VariableFunctionsClass.mean(data.float()).cpu().detach().float().numpy().tolist() + saved_tensor = data.contiguous().cpu().detach().numpy() + summary_data.extend([tensor_max, tensor_min, tensor_mean]) + return DataInfo(data, saved_tensor, summary_data, str(data.dtype), tuple(data.shape)) + + +def get_scalar_data_info(data): + summary_data = [data, data, data] + return DataInfo(data, data, summary_data, str(type(data)), str([])) + + +def get_float_tensor_info(data): + summary_data = [] + tensor_max = torch._C._VariableFunctionsClass.max(data).cpu().detach().float().numpy().tolist() + tensor_min = torch._C._VariableFunctionsClass.min(data).cpu().detach().float().numpy().tolist() + tensor_mean = torch._C._VariableFunctionsClass.mean(data).cpu().detach().float().numpy().tolist() + saved_tensor = data.contiguous().cpu().detach().numpy() + summary_data.extend([tensor_max, tensor_min, tensor_mean]) + return DataInfo(data, saved_tensor, summary_data, str(data.dtype), tuple(data.shape)) + + +def dump_tensor(args): + global data_info + args_list = [] + for x in args: + if isinstance(x, torch.Tensor): + if x.numel() == 0 or len(x.shape) == 0 or not x.is_floating_point(): + data_info = get_not_float_tensor_info(x) + else: + data_info = get_float_tensor_info(x) + arg = {"dtype": data_info.dtype, + "shape": data_info.shape, + "type": "torch.Tensor", + "Max": data_info.summary_data[0], + "Min": data_info.summary_data[1]} + else: + arg = {"value": None, + "type": type(x)} + args_list.append(arg) + return args_list + + +def dump_api_tensor(module, name_template, out_feat, dump_file): + api_params_dict = dict() + api_dict = dict() + if Const.BACKWARD in name_template and DumpUtil.dump_mode != Const.FORWARD: + path = os.path.dirname(dump_file) + dump_file = os.path.join(path, "dump_backward.pkl") + api_params_dict["args"] = dump_tensor(out_feat) + elif Const.BACKWARD not in name_template and DumpUtil.dump_mode != Const.BACKWARD: + if module.input_args: + args_list = dump_tensor(module.input_args) + api_params_dict["args"] = args_list + if module.input_kwargs: + api_params_dict["kwargs"] = module.input_kwargs + api_dict[name_template] = api_params_dict + with os.fdopen(os.open(dump_file, os.O_RDWR | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), + "a") as f: + json.dump(api_dict, f) + f.write('\n') + + +def dump_acc_cmp(name, out_feat, module): + dump_file = DumpUtil.get_dump_path() + _set_dump_switch4api_list(name) + + dump_file = modify_dump_path(dump_file, DumpUtil.dump_switch_mode) + + if DumpUtil.get_dump_switch(): + if DumpUtil.dump_init_enable: + DumpUtil.dump_init_enable = False + DumpUtil.dump_data_dir = make_dump_data_dir(dump_file) \ + if DumpUtil.dump_switch_mode not in [Const.STACK, Const.ACL] else "" + if os.path.exists(dump_file) and not os.path.isdir(dump_file): + check_writable(dump_file) + os.remove(dump_file) + + if DumpUtil.dump_switch_mode in [Const.ALL, Const.API_LIST]: + dump_api_tensor(module, name, out_feat, dump_file) + + + + +def acc_cmp_dump(name): + + def acc_cmp_hook(module, in_feat, out_feat): + dump_acc_cmp(name, out_feat, module) + if hasattr(module, "input_args"): + del module.input_args + if hasattr(module, "input_kwargs"): + del module.input_kwargs + + return acc_cmp_hook diff --git a/precision/api_ut_tools/dump/dump_scope.py b/precision/api_ut_tools/dump/dump_scope.py new file mode 100644 index 00000000000..51dbd75d9c8 --- /dev/null +++ b/precision/api_ut_tools/dump/dump_scope.py @@ -0,0 +1 @@ +# dump范围控制 ———— 李天 \ No newline at end of file diff --git a/precision/api_ut_tools/dump/info_dump.py b/precision/api_ut_tools/dump/info_dump.py new file mode 100644 index 00000000000..2cbd1750f1f --- /dev/null +++ b/precision/api_ut_tools/dump/info_dump.py @@ -0,0 +1 @@ +# 基于api——info信息，将其落盘为json文件 \ No newline at end of file diff --git a/precision/api_ut_tools/dump/utils.py b/precision/api_ut_tools/dump/utils.py new file mode 100644 index 00000000000..bebb693945a --- /dev/null +++ b/precision/api_ut_tools/dump/utils.py @@ -0,0 +1,219 @@ +import os +import shutil +import sys +from pathlib import Path + +from ..common.utils import print_error_log, CompareException, DumpException, Const, get_time, print_info_log, \ + check_mode_valid, get_api_name_from_matcher + +from ..common.version import __version__ + +dump_count = 0 +range_begin_flag, range_end_flag = False, False + + +class DumpUtil(object): + dump_data_dir = None + dump_path = None + dump_switch = None + dump_switch_mode = Const.ALL + dump_switch_scope = [] + dump_init_enable = False + dump_api_list = [] + dump_filter_switch = None + dump_mode = Const.ALL + backward_input = {} + dump_dir_tag = 'ptdbg_dump' + dump_config = None + + @staticmethod + def set_dump_path(save_path): + DumpUtil.dump_path = save_path + DumpUtil.dump_init_enable = True + + @staticmethod + def set_dump_config(dump_config): + DumpUtil.dump_config = dump_config + + @staticmethod + def set_dump_switch(switch, mode, scope, api_list, filter_switch, dump_mode): + DumpUtil.dump_switch = switch + DumpUtil.dump_switch_mode = mode + DumpUtil.dump_init_enable = True + DumpUtil.dump_switch_scope = scope + DumpUtil.dump_api_list = [api.lower() for api in api_list] + DumpUtil.dump_filter_switch = filter_switch + DumpUtil.dump_mode = dump_mode + if mode == Const.ACL: + DumpUtil.dump_switch_scope = [api_name.replace("backward", "forward") for api_name in scope] + + def check_list_or_acl_mode(name_prefix): + global dump_count + for item in DumpUtil.dump_switch_scope: + if name_prefix.startswith(item): + dump_count = dump_count + 1 + return True + + def check_range_mode(name_prefix): + global range_begin_flag + global range_end_flag + if name_prefix.startswith(DumpUtil.dump_switch_scope[0]): + range_begin_flag = True + return True + if name_prefix.startswith(DumpUtil.dump_switch_scope[1]): + range_end_flag = True + return True + if range_begin_flag and not range_end_flag: + return True + return False + + def check_stack_mode(name_prefix): + if len(DumpUtil.dump_switch_scope) == 0: + return True + elif len(DumpUtil.dump_switch_scope) == 1: + return name_prefix.startswith(DumpUtil.dump_switch_scope[0]) + elif len(DumpUtil.dump_switch_scope) == 2: + return DumpUtil.check_range_mode(name_prefix) + else: + print_error_log("dump scope is invalid, Please set the scope mode in" + " set_dump_switch with 'all', 'list', 'range', 'stack', 'acl', 'api_list'!") + return False + + check_mapper = { + Const.LIST: check_list_or_acl_mode, + Const.ACL: check_list_or_acl_mode, + Const.RANGE: check_range_mode, + Const.STACK: check_stack_mode + } + + @staticmethod + def check_switch_scope(name_prefix): + if DumpUtil.dump_switch_mode in DumpUtil.check_mapper: + check_func = DumpUtil.check_mapper[DumpUtil.dump_switch_mode] + return check_func(name_prefix) + return False + + @staticmethod + def get_dump_path(): + if DumpUtil.dump_path: + return DumpUtil.dump_path + + if DumpUtil.dump_switch_mode == Const.ALL: + raise RuntimeError("get_dump_path: the file path is empty," + " you must use set_dump_path to set a valid dump path!!!") + else: + dir_path = os.path.realpath("./") + dump_file_name = "scope_dump_{}_{}_{}.pkl".format( + DumpUtil.dump_switch_mode, DumpUtil.dump_switch_scope[0], get_time()) + DumpUtil.dump_path = os.path.join(dir_path, dump_file_name) + return DumpUtil.dump_path + + @staticmethod + def get_dump_switch(): + return DumpUtil.dump_switch == "ON" + + +def set_dump_path(fpath=None, dump_tag='ptdbg_dump'): + if fpath is None: + raise RuntimeError("set_dump_path '{}' error, please set a valid filename".format(fpath)) + return + real_path = os.path.realpath(fpath) + if not os.path.isdir(real_path): + print_error_log( + "set_dump_path '{}' error, the path is not a directory please set a valid directory.".format(real_path)) + raise DumpException(DumpException.INVALID_PATH_ERROR) + DumpUtil.set_dump_path(real_path) + DumpUtil.dump_dir_tag = dump_tag + + +def generate_dump_path_str(): + if DumpUtil.dump_switch_mode == 'acl': + if DumpUtil.dump_config == '': + print_error_log("Please provide dump config for register hook before turning on dump switch!") + raise DumpException(DumpException.NONE_ERROR) + dump_path = f"according to dump config {DumpUtil.dump_config}" + else: + dump_path = f"to {DumpUtil.dump_path}" + return dump_path + + +def set_dump_switch(switch, mode=Const.ALL, scope=[], api_list=[], filter_switch=Const.ON, dump_mode=Const.ALL): + try: + check_mode_valid(mode) + assert switch in ["ON", "OFF"], "Please set dump switch with 'ON' or 'OFF'." + assert filter_switch in ["ON", "OFF"], "Please set filter_switch with 'ON' or 'OFF'." + assert dump_mode in ["all", "forward", "backward"], "Please set dump_mode with 'all' or 'forward' or 'backward'." + if mode == Const.RANGE: + assert len(scope) == 2, "set_dump_switch, scope param set invalid, it's must be [start, end]." + if mode == Const.LIST: + assert len(scope) != 0, "set_dump_switch, scope param set invalid, it's should not be an empty list." + if mode == Const.STACK: + assert len(scope) <= 2, "set_dump_switch, scope param set invalid, it's must be [start, end] or []." + if mode == Const.ACL: + assert len(scope) == 1, "set_dump_switch, scope param set invalid, only one api name is supported in acl mode." + if mode == Const.API_LIST: + assert isinstance(api_list, list) and len(api_list) >= 1, \ + "Current dump mode is 'api_list', but the content of api_list parameter is empty or valid." + except (CompareException, AssertionError) as err: + print_error_log(str(err)) + sys.exit() + + if switch == "OFF": + dump_path_str = generate_dump_path_str() + DumpUtil.set_dump_switch(switch, mode=mode, scope=scope, api_list=api_list, filter_switch=filter_switch, dump_mode=dump_mode) + if switch == "ON": + dump_path_str = generate_dump_path_str() + + global dump_count + if switch == "ON": + print_info_log(f"Dump switch is turned on. Dump data will be saved {dump_path_str}. ") + if mode == Const.LIST: + dump_count = 0 + else: + print_info_log(f"Dump switch is turned off. Dump data has been saved {dump_path_str}. ") + if mode == Const.LIST: + print_info_log("The number of matched dump is {}".format(dump_count)) + +def _set_dump_switch4api_list(name): + if DumpUtil.dump_api_list: + api_name = get_api_name_from_matcher(name) + DumpUtil.dump_switch = "ON" if api_name in DumpUtil.dump_api_list else "OFF" + + +def set_backward_input(backward_input): + for index, api_name in enumerate(DumpUtil.dump_switch_scope): + DumpUtil.backward_input[api_name] = backward_input[index] + + +def make_dump_data_dir(dump_file_name): + dump_path, file_name = os.path.split(os.path.realpath(dump_file_name)) + name_body, name_extension = os.path.splitext(file_name) + output_dir = os.path.join(dump_path, f"{name_body}") + if not os.path.exists(output_dir): + os.mkdir(output_dir, mode=0o750) + else: + shutil.rmtree(output_dir, ignore_errors=True) + os.mkdir(output_dir, mode=0o750) + return output_dir + + +def make_dump_dirs(rank): + dump_file_name, dump_file_name_body = "dump.pkl", "dump" + dump_root_dir = DumpUtil.dump_path if DumpUtil.dump_path else "./" + tag_dir = os.path.join(dump_root_dir, DumpUtil.dump_dir_tag + f'_v{__version__}') + Path(tag_dir).mkdir(mode=0o750, parents=True, exist_ok=True) + rank_dir = os.path.join(tag_dir, 'rank' + str(rank)) + if not os.path.exists(rank_dir): + os.mkdir(rank_dir, mode=0o750) + DumpUtil.dump_dir = rank_dir + dump_file_path = os.path.join(rank_dir, dump_file_name) + DumpUtil.set_dump_path(dump_file_path) + + +def check_writable(dump_file): + if not os.access(dump_file, os.W_OK): + print_error_log( + 'The path {} does not have permission to write. Please check the path permission'.format( + dump_file)) + raise DumpException(DumpException.INVALID_PATH_ERROR) + diff --git a/precision/api_ut_tools/hook_module/.keep b/precision/api_ut_tools/hook_module/.keep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/precision/api_ut_tools/hook_module/hook_module.py b/precision/api_ut_tools/hook_module/hook_module.py new file mode 100644 index 00000000000..c18b37a9c9a --- /dev/null +++ b/precision/api_ut_tools/hook_module/hook_module.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + + +import functools + +import torch +import torch.nn as nn +import torch.utils.hooks as full_hooks + +module_count = {} +g_stop_hook = False + + +class HOOKModule(nn.Module): + + def __init__(self, hook) -> None: + super(HOOKModule, self).__init__() + self.has_overflow = False + self.input_args = tuple() + self.input_kwargs = dict() + self._enable_hook = True + prefix = "" + if hasattr(self, "prefix_op_name_"): + prefix = self.prefix_op_name_ + + if prefix not in module_count: + module_count[prefix] = 1 + prefix += '0*' + else: + module_count[prefix] += 1 + prefix = prefix + str(module_count[prefix] - 1) + '*' + + self.register_forward_hook(hook(prefix + "forward")) + self.register_backward_hook(hook(prefix + "backward")) + + def __call__(self, *input, **kwargs): + changed = False + global g_stop_hook + if g_stop_hook: + self._enable_hook = False + else: + g_stop_hook = True + changed = True + result = self._call_func(*input, **kwargs) + if changed: + g_stop_hook = False + return result + + def _call_func(self, *input, **kwargs): + if self._enable_hook: + full_backward_hooks, non_full_backward_hooks = [], [] + if len(self._backward_hooks) > 0: + full_backward_hooks, non_full_backward_hooks = self._get_backward_hooks() + for hook in self._forward_pre_hooks.values(): + result = hook(self, input) + if result is not None: + if not isinstance(result, tuple): + result = (result,) + input = result + bw_hook = None + if len(full_backward_hooks) > 0: + bw_hook = full_hooks.BackwardHook(self, full_backward_hooks) + input = bw_hook.setup_input_hook(input) + self.input_args = input + self.input_kwargs = kwargs + if torch._C._get_tracing_state(): + result = self._slow_forward(*input, **kwargs) + else: + result = self.forward(*input, **kwargs) + for hook in self._forward_hooks.values(): + hook_result = hook(self, input, result) + if hook_result is not None: + result = hook_result + if bw_hook: + result = bw_hook.setup_output_hook(result) + if len(non_full_backward_hooks) > 0: + var = result + while not isinstance(var, torch.Tensor): + if isinstance(var, dict): + var = next((v for v in var.values() if isinstance(v, torch.Tensor))) + elif isinstance(var, (list, tuple)): + if var: + var = var[0] + else: + return result + else: + return result + grad_fn = var.grad_fn + if grad_fn is not None: + for hook in non_full_backward_hooks: + wrapper = functools.partial(hook, self) + functools.update_wrapper(wrapper, hook) + grad_fn.register_hook(wrapper) + self._maybe_warn_non_full_backward_hook(input, result, grad_fn) + return result + else: + forward_call = (self._slow_forward if torch._C._get_tracing_state() else self.forward) + return forward_call(*input, **kwargs) diff --git a/precision/api_ut_tools/hook_module/register_hook.py b/precision/api_ut_tools/hook_module/register_hook.py new file mode 100644 index 00000000000..35d02bb641c --- /dev/null +++ b/precision/api_ut_tools/hook_module/register_hook.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +import functools +import os + +import torch + +from . import wrap_torch, wrap_functional, wrap_tensor +from .hook_module import HOOKModule +from ..common.utils import check_file_or_directory_path, print_error_log, CompareException, Const, \ + print_info_log, print_warn_log, get_process_rank +from ..dump.utils import make_dump_dirs +from ..dump.dump import acc_cmp_dump + +try: + import torch_npu +except ImportError: + is_gpu = True +else: + is_gpu = False + +make_dir_flag = True + + +def initialize_hook(hook): + wrap_tensor.wrap_tensor_ops_and_bind(hook) + for attr_name in dir(wrap_tensor.HOOKTensor): + if attr_name.startswith("wrap_"): + setattr(torch.Tensor, attr_name[5:], getattr(wrap_tensor.HOOKTensor, attr_name)) + + wrap_torch.wrap_torch_ops_and_bind(hook) + for attr_name in dir(wrap_torch.HOOKTorchOP): + if attr_name.startswith("wrap_"): + setattr(torch, attr_name[5:], getattr(wrap_torch.HOOKTorchOP, attr_name)) + + wrap_functional.wrap_functional_ops_and_bind(hook) + for attr_name in dir(wrap_functional.HOOKFunctionalOP): + if attr_name.startswith("wrap_"): + setattr(torch.nn.functional, attr_name[5:], getattr(wrap_functional.HOOKFunctionalOP, attr_name)) + + +def register_hook(): + global make_dir_flag + if make_dir_flag: + make_dump_dirs(0) + make_dir_flag = False + initialize_hook(acc_cmp_dump) diff --git a/precision/api_ut_tools/hook_module/support_wrap_ops.yaml b/precision/api_ut_tools/hook_module/support_wrap_ops.yaml new file mode 100644 index 00000000000..6aefa1460b3 --- /dev/null +++ b/precision/api_ut_tools/hook_module/support_wrap_ops.yaml @@ -0,0 +1,1000 @@ +# Copyright (c) 2020 Huawei Technologies Co., Ltd +# All rights reserved. +# +# Licensed under the BSD 3-Clause License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# List of ops that register hooks + +functional: + - conv1d + - conv2d + - conv3d + - conv_transpose1d + - conv_transpose2d + - conv_transpose3d + - conv_tbc + - avg_pool1d + - avg_pool2d + - avg_pool3d + - fractional_max_pool2d_with_indices + - fractional_max_pool2d + - fractional_max_pool3d_with_indices + - fractional_max_pool3d + - max_pool1d_with_indices + - max_pool1d + - max_pool2d_with_indices + - max_pool2d + - max_pool3d_with_indices + - max_pool3d + - max_unpool1d + - max_unpool2d + - max_unpool3d + - lp_pool2d + - lp_pool1d + - adaptive_max_pool1d_with_indices + - adaptive_max_pool1d + - adaptive_max_pool2d_with_indices + - adaptive_max_pool2d + - adaptive_max_pool3d_with_indices + - adaptive_max_pool3d + - adaptive_avg_pool1d + - adaptive_avg_pool2d + - adaptive_avg_pool3d + - dropout + - alpha_dropout + - dropout2d + - dropout3d + - feature_alpha_dropout + - threshold + - threshold_ + - relu + - relu_ + - glu + - hardtanh + - hardtanh_ + - relu6 + - elu + - elu_ + - selu + - selu_ + - celu + - celu_ + - leaky_relu + - leaky_relu_ + - prelu + - rrelu + - rrelu_ + - logsigmoid + - gelu + - hardshrink + - tanhshrink + - softsign + - softplus + - softmin + - softmax + - gumbel_softmax + - log_softmax + - softshrink + - tanh + - sigmoid + - hardsigmoid + - linear + - bilinear + - silu + - hardswish + - embedding + - embedding_bag + - batch_norm + - instance_norm + - layer_norm + - group_norm + - local_response_norm + - ctc_loss + - nll_loss + - poisson_nll_loss + - gaussian_nll_loss + - kl_div + - cross_entropy + - binary_cross_entropy + - binary_cross_entropy_with_logits + - smooth_l1_loss + - l1_loss + - mse_loss + - margin_ranking_loss + - hinge_embedding_loss + - multilabel_margin_loss + - soft_margin_loss + - multilabel_soft_margin_loss + - cosine_embedding_loss + - multi_margin_loss + - pixel_shuffle + - pixel_unshuffle + - channel_shuffle + - upsample + - interpolate + - upsample_nearest + - upsample_bilinear + - grid_sample + - affine_grid + - pad + - pairwise_distance + - pdist + - cosine_similarity + - one_hot + - triplet_margin_loss + - triplet_margin_with_distance_loss + - normalize + - unfold + - fold + - multi_head_attention_forward + +tensor: + - __add__ + - __and__ + - __bool__ + - __div__ + - __eq__ + - __ge__ + - __gt__ + - __getitem__ + - __iadd__ + - __iand__ + - __idiv__ + - __ifloordiv__ + - __ilshift__ + - __imod__ + - __imul__ + - __ior__ + - __irshift__ + - __isub__ + - __ixor__ + - __lshift__ + - __matmul__ + - __mod__ + - __mul__ + - __nonzero__ + - __or__ + - __radd__ + - __rmul__ + - __rshift__ + - __sub__ + - __truediv__ + - __xor__ + - abs + - abs_ + - absolute + - absolute_ + - acos + - acos_ + - acosh + - acosh_ + - add + - add_ + - addbmm + - addbmm_ + - addcdiv + - addcdiv_ + - addcmul + - addcmul_ + - addmm + - addmm_ + - addmv + - addmv_ + - addr + - addr_ + - align_as + - align_to + - all + - allclose + - amax + - amin + - angle + - any + - arccos + - arccos_ + - arccosh + - arccosh_ + - arcsin + - arcsin_ + - arcsinh + - arcsinh_ + - arctan + - arctan_ + - arctanh + - arctanh_ + - argmax + - argmin + - argsort + - asin + - asin_ + - asinh + - asinh_ + - atan + - atan2 + - atan2_ + - atan_ + - atanh + - atanh_ + - baddbmm + - baddbmm_ + - bernoulli + - bernoulli_ + - bincount + - bitwise_and + - bitwise_and_ + - bitwise_not + - bitwise_not_ + - bitwise_or + - bitwise_or_ + - bitwise_xor + - bitwise_xor_ + - bmm + - broadcast_to + - cauchy_ + - ceil + - ceil_ + - cholesky + - chunk + - clamp + - cholesky_solve + - cholesky_inverse + - clamp_ + - clamp_max + - clamp_max_ + - clip + - clamp_min + - clamp_min_ + - clip_ + - copysign + - copysign_ + - cos + - cos_ + - cosh + - cosh_ + - count_nonzero + - cummax + - cummin + - cumprod + - cumprod_ + - cumsum + - cumsum_ + - deg2rad + - deg2rad_ + - det + - diag + - diag_embed + - diagflat + - diagonal + - diff + - dist + - digamma + - digamma_ + - div + - div_ + - divide + - divide_ + - dot + - eig + - eq + - eq_ + - erf + - equal + - erf_ + - erfc + - erfc_ + - erfinv + - erfinv_ + - exp + - exp2 + - exp2_ + - expm1 + - exp_ + - expm1_ + - exponential_ + - fill_ + - fix + - fill_diagonal_ + - fix_ + - flip + - fliplr + - flatten + - flipud + - float_power + - float_power_ + - floor + - floor_ + - floor_divide + - floor_divide_ + - fmax + - fmin + - fmod + - fmod_ + - frac + - frac_ + - gather + - gcd + - gcd_ + - ge + - ge_ + - geometric_ + - geqrf + - ger + - greater + - greater_ + - gt + - gt_ + - greater_equal + - greater_equal_ + - hardshrink + - heaviside + - heaviside_ + - histc + - hypot + - hypot_ + - igamma + - igamma_ + - igammac + - igammac_ + - index_add + - index_add_ + - inverse + - index_copy + - index_copy_ + - index_fill + - index_fill_ + - index_put + - index_put_ + - inner + - index_select + - isclose + - isfinite + - isinf + - isnan + - isneginf + - isposinf + - isreal + - kron + - kthvalue + - lcm + - lcm_ + - ldexp + - ldexp_ + - le + - le_ + - lerp + - lerp_ + - where + - less + - less_ + - less_equal + - less_equal_ + - lgamma + - lgamma_ + - log + - log10 + - log10_ + - log1p + - log1p_ + - log2 + - log2_ + - log_ + - log_normal_ + - log_softmax + - logcumsumexp + - logdet + - logaddexp + - logaddexp2 + - logical_and + - logical_and_ + - logical_not + - logit + - logical_not_ + - logical_or + - logical_or_ + - logical_xor + - logical_xor_ + - logit_ + - logsumexp + - lstsq + - lt + - lt_ + - lu_solve + - map2_ + - map_ + - masked_fill + - matmul + - masked_fill_ + - masked_scatter + - masked_scatter_ + - masked_select + - matrix_exp + - max + - maximum + - mean + - matrix_power + - median + - min + - minimum + - mm + - mode + - msort + - mul + - mul_ + - multinomial + - multiply + - multiply_ + - mv + - mvlgamma + - mvlgamma_ + - nansum + - narrow + - narrow_copy + - ne + - ne_ + - neg + - neg_ + - negative + - negative_ + - nonzero + - normal_ + - not_equal + - not_equal_ + - permute + - pinverse + - polygamma + - pow + - pow_ + - polygamma_ + - prelu + - prod + - put_ + - rad2deg + - rad2deg_ + - ravel + - real + - reciprocal + - reciprocal_ + - relu + - relu_ + - remainder + - repeat_interleave + - reshape + - remainder_ + - renorm + - renorm_ + - repeat + - reshape_as + - resize_ + - resize_as_ + - roll + - rot90 + - round + - round_ + - rsqrt + - rsqrt_ + - scatter + - scatter_ + - scatter_add + - scatter_add_ + - select + - sgn + - sgn_ + - sigmoid + - sigmoid_ + - sign + - sign_ + - signbit + - sin + - sin_ + - sinc + - sinc_ + - sinh + - sinh_ + - slogdet + - smm + - softmax + - solve + - sort + - split_with_sizes + - sqrt + - sqrt_ + - square + - square_ + - squeeze + - squeeze_ + - sspaddmm + - std + - sub + - sub_ + - sum + - sum_to_size + - svd + - symeig + - t + - t_ + - take + - tan + - tan_ + - tanh + - tanh_ + - tensor_split + - tile + - topk + - transpose + - transpose_ + - triangular_solve + - tril + - tril_ + - triu + - true_divide + - triu_ + - true_divide_ + - trunc + - trunc_ + - type_as + - unbind + - unflatten + - unfold + - unsafe_chunk + - unsqueeze + - unsafe_split + - unsafe_split_with_sizes + - var + - vdot + - unsqueeze_ + - view_as + - xlogy + - xlogy_ + +torch: + - _adaptive_avg_pool2d + - _add_relu + - _add_relu_ + - _aminmax + - _batch_norm_impl_index + - _convolution + - abs + - abs_ + - absolute + - acos + - acos_ + - acosh + - acosh_ + - adaptive_avg_pool1d + - adaptive_max_pool1d + - add + - addbmm + - addcdiv + - addcmul + - addmm + - addmv + - addmv_ + - addr + - amax + - affine_grid_generator + - align_tensors + - all + - alpha_dropout + - amin + - alpha_dropout_ + - angle + - any + - arange + - arccos + - arccos_ + - arccosh + - arccosh_ + - arcsin + - arcsin_ + - arcsinh + - arcsinh_ + - arctan + - arctan_ + - arctanh + - arctanh_ + - argmax + - argmin + - argsort + - asin + - asin_ + - asinh + - asinh_ + - atan + - atan2 + - atan_ + - atanh + - atanh_ + - atleast_1d + - atleast_2d + - atleast_3d + - avg_pool1d + - baddbmm + - bartlett_window + - batch_norm_backward_elemt + - batch_norm_backward_reduce + - batch_norm_elemt + - batch_norm_gather_stats + - batch_norm_gather_stats_with_counts + - bernoulli + - batch_norm_stats + - batch_norm_update_stats + - bilinear + - bincount + - binomial + - binary_cross_entropy_with_logits + - bitwise_and + - bitwise_not + - bitwise_or + - bitwise_xor + - blackman_window + - block_diag + - bmm + - broadcast_tensors + - broadcast_to + - cartesian_prod + - cat + - cdist + - ceil + - ceil_ + - celu + - celu_ + - chain_matmul + - channel_shuffle + - cholesky + - cholesky_inverse + - cholesky_solve + - choose_qparams_optimized + - chunk + - clamp + - clamp_ + - clamp_max + - clamp_max_ + - clamp_min + - clamp_min_ + - clip + - clip_ + - clone + - column_stack + - combinations + - constant_pad_nd + - conv1d + - conv2d + - conv3d + - conv_tbc + - conv_transpose1d + - conv_transpose2d + - conv_transpose3d + - cos + - convolution + - copysign + - cos_ + - cosh + - cosh_ + - cosine_embedding_loss + - cosine_similarity + - count_nonzero + - cross + - ctc_loss + - cummax + - cummin + - cumprod + - cumsum + - deg2rad + - deg2rad_ + - det + - diag + - diag_embed + - diff + - diagflat + - diagonal + - digamma + - dist + - div + - divide + - dot + - dropout + - dropout_ + - dsmm + - dstack + - eig + - einsum + - embedding + - embedding_bag + - embedding_renorm_ + - eq + - equal + - erf + - erf_ + - erfc + - erfc_ + - erfinv + - exp + - exp2 + - exp2_ + - exp_ + - expm1 + - expm1_ + - eye + - feature_dropout + - feature_alpha_dropout + - feature_alpha_dropout_ + - feature_dropout_ + - fix + - fill_ + - fix_ + - flatten + - flip + - fliplr + - flipud + - float_power + - floor + - floor_ + - floor_divide + - fmax + - fmin + - fmod + - frac + - frac_ + - full + - frobenius_norm + - full_like + - gather + - gcd + - gcd_ + - ge + - geqrf + - ger + - greater + - greater_equal + - grid_sampler + - grid_sampler_2d + - group_norm + - grid_sampler_3d + - gru + - gru_cell + - gt + - hamming_window + - hann_window + - hardshrink + - heaviside + - hinge_embedding_loss + - histc + - hsmm + - hspmm + - hstack + - hypot + - igamma + - igammac + - index_add + - index_copy + - inner + - index_fill + - index_put + - index_put_ + - index_select + - instance_norm + - isclose + - isfinite + - isinf + - isnan + - isneginf + - isposinf + - istft + - kaiser_window + - kl_div + - kron + - kthvalue + - layer_norm + - lcm + - lcm_ + - ldexp + - ldexp_ + - le + - lerp + - less + - less_equal + - lgamma + - linspace + - log + - log10 + - log10_ + - log1p + - log1p_ + - log2 + - log2_ + - log_softmax + - log_ + - logaddexp + - logaddexp2 + - logcumsumexp + - logdet + - logical_and + - logical_not + - logical_or + - logical_xor + - logit + - logit_ + - logspace + - logsumexp + - lstm + - lstm_cell + - lstsq + - lt + - lu_solve + - masked_fill + - margin_ranking_loss + - masked_scatter + - masked_select + - matrix_exp + - matmul + - matrix_power + - matrix_rank + - max + - max_pool1d + - max_pool2d + - max_pool1d_with_indices + - max_pool3d + - maximum + - mean + - median + - min + - minimum + - mm + - mode + - moveaxis + - movedim + - msort + - mul + - multinomial + - multiply + - mv + - mvlgamma + - nan_to_num + - nan_to_num_ + - nanmedian + - nansum + - narrow + - native_batch_norm + - native_group_norm + - narrow_copy + - native_layer_norm + - native_norm + - ne + - neg + - negative + - neg_ + - negative_ + - nextafter + - nonzero + - norm_except_dim + - normal + - not_equal + - nuclear_norm + - pairwise_distance + - pdist + - pinverse + - pixel_shuffle + - pixel_unshuffle + - poisson + - poisson_nll_loss + - polar + - polygamma + - pow + - prelu + - prod + - rad2deg + - promote_types + - rad2deg_ + - range + - ravel + - real + - reciprocal + - relu + - reciprocal_ + - relu_ + - remainder + - renorm + - repeat_interleave + - reshape + - resize_as_ + - roll + - rot90 + - round + - round_ + - rrelu + - rrelu_ + - rsqrt + - row_stack + - rsqrt_ + - rsub + - saddmm + - scalar_tensor + - scatter + - select + - scatter_add + - searchsorted + - selu + - selu_ + - sgn + - sigmoid + - sigmoid_ + - sign + - signbit + - sin + - sin_ + - sinc + - sinc_ + - sinh + - sinh_ + - slogdet + - smm + - softmax + - solve + - sort + - sparse_coo_tensor + - square + - split_with_sizes + - spmm + - sqrt + - sqrt_ + - square_ + - squeeze + - sspaddmm + - stack + - std + - std_mean + - sub + - subtract + - sum + - svd + - swapaxes + - swapdims + - symeig + - t + - take + - tan + - tan_ + - tanh + - tanh_ + - tensordot + - tensor_split + - threshold + - threshold_ + - tile + - topk + - transpose + - trapz + - triangular_solve + - tril + - tril_indices + - triplet_margin_loss + - triu + - triu_indices + - true_divide + - trunc + - trunc_ + - unique_consecutive + - xlogy + - unbind + - unique_dim + - unsafe_chunk + - unsafe_split + - vander + - var + - vdot + - unsafe_split_with_sizes + - unsqueeze + - var_mean + - vstack + - where + - xlogy_ diff --git a/precision/api_ut_tools/hook_module/wrap_functional.py b/precision/api_ut_tools/hook_module/wrap_functional.py new file mode 100644 index 00000000000..0eff94c3413 --- /dev/null +++ b/precision/api_ut_tools/hook_module/wrap_functional.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +import os + +import torch +import yaml + +from .hook_module import HOOKModule +from ..common.utils import torch_device_guard + +cur_path = os.path.dirname(os.path.realpath(__file__)) +yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") +with open(yaml_path, 'r') as f: + WrapFunctionalOps = yaml.safe_load(f).get('functional') + +for f in dir(torch.nn.functional): + locals().update({f: getattr(torch.nn.functional, f)}) + + +def get_functional_ops(): + global WrapFunctionalOps + _all_functional_ops = dir(torch.nn.functional) + return set(WrapFunctionalOps) & set(_all_functional_ops) + + +class HOOKFunctionalOP(object): + pass + + +class FunctionalOPTemplate(HOOKModule): + def __init__(self, op_name, hook): + self.op_name_ = op_name + self.prefix_op_name_ = "Functional*" + str(op_name) + "*" + super().__init__(hook) + + @torch_device_guard + def forward(self, *args, **kwargs): + return eval(self.op_name_)(*args, **kwargs) + + +def wrap_functional_op(op_name, hook): + def functional_op_template(*args, **kwargs): + return FunctionalOPTemplate(op_name, hook)(*args, **kwargs) + + return functional_op_template + + +def wrap_functional_ops_and_bind(hook): + _functional_ops = get_functional_ops() + for op_name in _functional_ops: + setattr(HOOKFunctionalOP, "wrap_" + op_name, wrap_functional_op(op_name, hook)) diff --git a/precision/api_ut_tools/hook_module/wrap_tensor.py b/precision/api_ut_tools/hook_module/wrap_tensor.py new file mode 100644 index 00000000000..07fbca8711d --- /dev/null +++ b/precision/api_ut_tools/hook_module/wrap_tensor.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +import os + +import torch +import yaml + +from .hook_module import HOOKModule +from ..common.utils import torch_device_guard + +cur_path = os.path.dirname(os.path.realpath(__file__)) +yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") +with open(yaml_path, 'r') as f: + WrapTensorOps = yaml.safe_load(f).get('tensor') + + +def get_tensor_ops(): + global WrapTensorOps + _tensor_ops = dir(torch._C._TensorBase) + return set(WrapTensorOps) & set(_tensor_ops) + + +class HOOKTensor(object): + pass + + +class TensorOPTemplate(HOOKModule): + + def __init__(self, op_name, hook): + self.op_name_ = op_name + self.prefix_op_name_ = "Tensor*" + str(op_name) + "*" + super().__init__(hook) + + @torch_device_guard + def forward(self, *args, **kwargs): + return getattr(torch._C._TensorBase, str(self.op_name_))(*args, **kwargs) + + +def wrap_tensor_op(op_name, hook): + + def tensor_op_template(*args, **kwargs): + return TensorOPTemplate(op_name, hook)(*args, **kwargs) + + return tensor_op_template + + +def wrap_tensor_ops_and_bind(hook): + _tensor_ops = get_tensor_ops() + for op_name in _tensor_ops: + setattr(HOOKTensor, "wrap_" + str(op_name), wrap_tensor_op(op_name, hook)) diff --git a/precision/api_ut_tools/hook_module/wrap_torch.py b/precision/api_ut_tools/hook_module/wrap_torch.py new file mode 100644 index 00000000000..23334be233f --- /dev/null +++ b/precision/api_ut_tools/hook_module/wrap_torch.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +import os + +import torch +import yaml + +from .hook_module import HOOKModule +from ..common.utils import torch_device_guard + +cur_path = os.path.dirname(os.path.realpath(__file__)) +yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") +with open(yaml_path, 'r') as f: + WrapTorchOps = yaml.safe_load(f).get('torch') + + +def get_torch_ops(): + global WrapTorchOps + _torch_ops = dir(torch._C._VariableFunctionsClass) + return set(WrapTorchOps) & set(_torch_ops) + + +class HOOKTorchOP(object): + pass + + +class TorchOPTemplate(HOOKModule): + + def __init__(self, op_name, hook): + self.op_name_ = op_name + self.prefix_op_name_ = "Torch_" + str(op_name) + "_" + super().__init__(hook) + + def input_param_need_adapt(self): + special_op_list = ["broadcast_tensors"] + for item in special_op_list: + if item in self.op_name_: + return True + return False + + def einsum_adapt(self, *args): + if len(args) < 2: + raise ValueError('einsum(): must specify the equation string and at least one operand, ' + 'or at least one operand and its subscripts list') + equation = None + operands = None + if isinstance(args[0], torch.Tensor): + def parse_subscript(n: int) -> str: + if n == Ellipsis: + return '...' + if n >= 0 and n < 26: + return chr(ord('A') + n) + if n >= 26 and n < 52: + return chr(ord('a') + n - 26) + raise ValueError('einsum(): subscript in subscript list is not within the valid range [0, 52]') + equation = ','.join(''.join(parse_subscript(s) for s in l) for l in args[1::2]) + + if len(args) % 2 == 1: + equation += '->' + ''.join(parse_subscript(s) for s in args[-1]) + operands = args[:-1:2] + else: + operands = args[::2] + else: + equation = args[0] + operands = args[1:] + + if len(operands) == 1 and isinstance(operands[0], (list, tuple)): + _operands = operands[0] + return self.einsum_adapt(equation, *_operands) + return equation, operands + + @torch_device_guard + def forward(self, *args, **kwargs): + if self.input_param_need_adapt(): + return getattr(torch._C._VariableFunctionsClass, str(self.op_name_))(args, **kwargs) + else: + if self.op_name_ == 'einsum': + args = self.einsum_adapt(*args) + return getattr(torch._C._VariableFunctionsClass, str(self.op_name_))(*args, **kwargs) + + +def wrap_torch_op(op_name, hook): + + def torch_op_template(*args, **kwargs): + return TorchOPTemplate(op_name, hook)(*args, **kwargs) + + return torch_op_template + + +def wrap_torch_ops_and_bind(hook): + _torch_ops = get_torch_ops() + for op_name in _torch_ops: + setattr(HOOKTorchOP, "wrap_" + op_name, wrap_torch_op(op_name, hook)) diff --git a/precision/api_ut_tools/run_ut/.keep b/precision/api_ut_tools/run_ut/.keep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/precision/api_ut_tools/run_ut/data_generate.py b/precision/api_ut_tools/run_ut/data_generate.py new file mode 100644 index 00000000000..5976d90049e --- /dev/null +++ b/precision/api_ut_tools/run_ut/data_generate.py @@ -0,0 +1 @@ +# 用于解析落盘json，进行数据生成 \ No newline at end of file diff --git a/precision/api_ut_tools/run_ut/run_ut.py b/precision/api_ut_tools/run_ut/run_ut.py new file mode 100644 index 00000000000..1c964ac5f31 --- /dev/null +++ b/precision/api_ut_tools/run_ut/run_ut.py @@ -0,0 +1,92 @@ +# 用户构造并运行api用例，注意前反向的区分 +import yaml +import os +import json +import torch + +FLOAT_TYPE = ['torch.float32', 'torch.float', 'torch.float64', 'torch.double', 'torch.float16', \ + 'torch.half', 'torch.bfloat16'] + +cur_path = os.path.dirname(os.path.realpath(__file__)) +yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") +with open(yaml_path, 'r') as f: + WrapFunctionalOps = yaml.safe_load(f).get('functional') + +for f in dir(torch.nn.functional): + locals().update({f: getattr(torch.nn.functional, f)}) + + +def run_ut(): + print("start") + forward_pkl = open("/home/wangchao/torch_test/dump_data_new/npu/ptdbg_dump_v1.0/rank0/dump.pkl") + backward_pkl = open("/home/wangchao/torch_test/dump_data_new/npu/ptdbg_dump_v1.0/rank0/dump_backward.pkl") + forward_content = forward_pkl.readlines() + backward_content = backward_pkl.readlines() + for api_info in forward_content: + api_json = json.loads(api_info) + for key, value in api_json.items(): + [api_type, api_name, index, mode] = key.split("*") + print(api_name) + api_feature = key.rsplit("*", 1)[0] + args, kwargs = generate_input(value.get("args"), api_json.get("kwargs")) + if api_type == "Functional": + out = eval(api_name)(*args, **kwargs) + if api_type == "Tensor": + out = getattr(torch._C._TensorBase, str(api_name))(*args, **kwargs) + for line in backward_content: + if api_feature in line: + api_back_json = json.loads(line) + for params in api_back_json.values(): + grad = nested_generate_input(params.get("args"), True, False) + out.backward(grad) + input_grad = [tensor.grad for tensor in args if isinstance(tensor, torch.Tensor)] + print("forward") + print(out) + print("backward") + print(input_grad) + + +def generate_input(input_args, input_kwargs, need_backward=True, need_convert=False): # 没有考虑dict of tensor + args = [] + kwargs = {} + + for info in input_args: + args.append(nested_generate_input(info, need_backward, need_convert)) + if kwargs: + for key, info in input_kwargs.items(): + kwargs[key] = nested_generate_input(info, need_backward, need_convert) + return args, kwargs + + +def nested_generate_input(info, need_backward, need_convert): + if isinstance(info, list): + result = [] + for i in info: + result.append(nested_generate_input(i, need_backward, need_convert)) + return result + # return list(map(nested_generate_input, info)) + # elif isinstance(input_info, tuple): + # return tuple(map(generate_input, input_info)) + else: + if info['type'] == 'torch.Tensor': + low, high = info['Min'], info['Max'] + data_dtype = info['dtype'] + if data_dtype in FLOAT_TYPE: #应该搞个float类型列表 + if need_convert and data_dtype == "torch.float16": + data_dtype = "torch.float32" + scale = high - low + rand01 = torch.rand(tuple(info['shape']), dtype=eval(data_dtype)) + inpt = rand01 * scale + low + if need_backward: + inpt.requires_grad_(True) + inpt.retain_grad() + elif 'int' in data_dtype or 'long' in data_dtype: # 应该搞个int类型列表, + inpt = torch.randint(int(low), int(high)+1, tuple(info['shape']), dtype=eval(data_dtype)) # high + 1因为右边是开区间 + else: + print(f'Warning: Dtype is not supported: ', info['dtype']) + raise NotImplementedError() + else: + inpt = info['value'] # 遗留问题：需要考虑是否要转换成原本类型 + return inpt + +run_ut() \ No newline at end of file -- Gitee From 8538e3bcc5daa32755121eb4d885b8abd55b3be2 Mon Sep 17 00:00:00 2001 From: wangchao Date: Fri, 21 Jul 2023 14:57:34 +0800 Subject: [PATCH 18/19] api ut tools first commit --- debug/tools/api_ut_tools/.keep | 0 debug/tools/api_ut_tools/__init__.py | 27 - debug/tools/api_ut_tools/common/.keep | 0 debug/tools/api_ut_tools/common/utils.py | 458 -------- debug/tools/api_ut_tools/dump/.keep | 0 debug/tools/api_ut_tools/dump/dump.py | 158 --- debug/tools/api_ut_tools/dump/utils.py | 219 ---- debug/tools/api_ut_tools/hook_module/.keep | 0 .../api_ut_tools/hook_module/hook_module.py | 113 -- .../api_ut_tools/hook_module/register_hook.py | 62 - .../hook_module/support_wrap_ops.yaml | 1000 ----------------- .../hook_module/wrap_functional.py | 66 -- .../api_ut_tools/hook_module/wrap_tensor.py | 65 -- .../api_ut_tools/hook_module/wrap_torch.py | 108 -- debug/tools/api_ut_tools/run_ut/.keep | 0 debug/tools/api_ut_tools/run_ut/run_ut.py | 91 -- 16 files changed, 2367 deletions(-) delete mode 100644 debug/tools/api_ut_tools/.keep delete mode 100644 debug/tools/api_ut_tools/__init__.py delete mode 100644 debug/tools/api_ut_tools/common/.keep delete mode 100644 debug/tools/api_ut_tools/common/utils.py delete mode 100644 debug/tools/api_ut_tools/dump/.keep delete mode 100644 debug/tools/api_ut_tools/dump/dump.py delete mode 100644 debug/tools/api_ut_tools/dump/utils.py delete mode 100644 debug/tools/api_ut_tools/hook_module/.keep delete mode 100644 debug/tools/api_ut_tools/hook_module/hook_module.py delete mode 100644 debug/tools/api_ut_tools/hook_module/register_hook.py delete mode 100644 debug/tools/api_ut_tools/hook_module/support_wrap_ops.yaml delete mode 100644 debug/tools/api_ut_tools/hook_module/wrap_functional.py delete mode 100644 debug/tools/api_ut_tools/hook_module/wrap_tensor.py delete mode 100644 debug/tools/api_ut_tools/hook_module/wrap_torch.py delete mode 100644 debug/tools/api_ut_tools/run_ut/.keep delete mode 100644 debug/tools/api_ut_tools/run_ut/run_ut.py diff --git a/debug/tools/api_ut_tools/.keep b/debug/tools/api_ut_tools/.keep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/debug/tools/api_ut_tools/__init__.py b/debug/tools/api_ut_tools/__init__.py deleted file mode 100644 index 9b62b9512de..00000000000 --- a/debug/tools/api_ut_tools/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -""" - -from .dump.dump import acc_cmp_dump -from .dump.utils import set_dump_path, set_dump_switch, set_backward_input -from .hook_module.register_hook import register_hook -from .common.utils import seed_all -from .common.version import __version__ -seed_all() - -__all__ = ["register_hook", "set_dump_path", "set_dump_switch", "seed_all", - "acc_cmp_dump"] diff --git a/debug/tools/api_ut_tools/common/.keep b/debug/tools/api_ut_tools/common/.keep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/debug/tools/api_ut_tools/common/utils.py b/debug/tools/api_ut_tools/common/utils.py deleted file mode 100644 index 39794456099..00000000000 --- a/debug/tools/api_ut_tools/common/utils.py +++ /dev/null @@ -1,458 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -import collections -import os -import random -import re -import stat -import subprocess -import sys -import time -from datetime import datetime, timezone - -import numpy as np -import torch - -try: - import torch_npu -except ImportError: - is_gpu = True -else: - is_gpu = False - -if not is_gpu: - from torch_npu.utils.device_guard import torch_device_guard as torch_npu_device_guard - -device = collections.namedtuple('device', ['type', 'index']) - - -class Const: - """ - Class for const - """ - MODEL_TYPE = ['.onnx', '.pb', '.om'] - DIM_PATTERN = r"^(-?[0-9]+)(,-?[0-9]+)*" - SEMICOLON = ";" - COLON = ":" - EQUAL = "=" - COMMA = "," - DOT = "." - DUMP_RATIO_MAX = 100 - SUMMERY_DATA_NUMS = 256 - ONE_HUNDRED_MB = 100*1024*1024 - FLOAT_EPSILON = np.finfo(float).eps - SUPPORT_DUMP_MODE = ['api', 'acl'] - ON = 'ON' - OFF = 'OFF' - BACKWARD = 'backward' - FORWARD = 'forward' - - # dump mode - ALL = "all" - LIST = "list" - RANGE = "range" - STACK = "stack" - ACL = "acl" - API_LIST = "api_list" - API_STACK = "api_stack" - DUMP_MODE = [ALL, LIST, RANGE, STACK, ACL, API_LIST, API_STACK] - - API_PATTERN = r"^[A-Za-z0-9]+[_]+([A-Za-z0-9]+[_]*[A-Za-z0-9]+)[_]+[0-9]+[_]+[A-Za-z0-9]+" - WRITE_FLAGS = os.O_WRONLY | os.O_CREAT - WRITE_MODES = stat.S_IWUSR | stat.S_IRUSR - - -class CompareConst: - """ - Class for compare module const - """ - # compare result column name - NPU_NAME = "NPU Name" - BENCH_NAME = "Bench Name" - NPU_DTYPE = "NPU Tensor Dtype" - BENCH_DTYPE = "Bench Tensor Dtype" - NPU_SHAPE = "NPU Tensor Shape" - BENCH_SHAPE = "Bench Tensor Shape" - NPU_MAX = "NPU max" - NPU_MIN = "NPU min" - NPU_MEAN = "NPU mean" - BENCH_MAX = "Bench max" - BENCH_MIN = "Bench min" - BENCH_MEAN = "Bench mean" - COSINE = "Cosine" - MAX_ABS_ERR = "MaxAbsErr" - ACCURACY = "Accuracy Reached or Not" - STACK = "NPU_Stack_Info" - ERROR_MESSAGE = "Err_message" - - # compare result data - NAN = 'Nan' - SHAPE_UNMATCH = 'shape unmatched' - DTYPE_UNMATCH = 'dtype unmatched' - - # accuracy standards - COS_THRESHOLD = 0.99 - MAX_ABS_ERR_THRESHOLD = 0.001 - COS_MAX_THRESHOLD = 0.9 - MAX_ABS_ERR_MAX_THRESHOLD = 1 - ACCURACY_CHECK_YES = "Yes" - ACCURACY_CHECK_NO = "No" - ACCURACY_CHECK_UNMATCH = "Unmatched" - - # error message - NO_BENCH = "No bench data matched." - - -class VersionCheck: - """ - Class for TorchVersion - """ - V1_8 = "1.8" - V1_11 = "1.11" - - @staticmethod - def check_torch_version(version): - torch_version = torch.__version__ - if torch_version.startswith(version): - return True - else: - return False - - -class CompareException(Exception): - """ - Class for Accuracy Compare Exception - """ - NONE_ERROR = 0 - INVALID_PATH_ERROR = 1 - OPEN_FILE_ERROR = 2 - CLOSE_FILE_ERROR = 3 - READ_FILE_ERROR = 4 - WRITE_FILE_ERROR = 5 - INVALID_FILE_ERROR = 6 - PERMISSION_ERROR = 7 - INDEX_OUT_OF_BOUNDS_ERROR = 8 - NO_DUMP_FILE_ERROR = 9 - INVALID_DATA_ERROR = 10 - INVALID_PARAM_ERROR = 11 - INVALID_DUMP_RATIO = 12 - INVALID_DUMP_FILE = 13 - UNKNOWN_ERROR = 14 - INVALID_DUMP_MODE = 15 - PARSE_FILE_ERROR = 16 - INVALID_COMPARE_MODE = 17 - - def __init__(self, code, error_info: str = ""): - super(CompareException, self).__init__() - self.code = code - self.error_info = error_info - - def __str__(self): - return self.error_info - -class DumpException(CompareException): - pass - -def _print_log(level, msg): - current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))) - pid = os.getgid() - print(current_time + "(" + str(pid) + ")-[" + level + "]" + msg) - sys.stdout.flush() - - -def print_info_log(info_msg): - """ - Function Description: - print info log. - Parameter: - info_msg: the info message. - """ - _print_log("INFO", info_msg) - - -def print_error_log(error_msg): - """ - Function Description: - print error log. - Parameter: - error_msg: the error message. - """ - _print_log("ERROR", error_msg) - - -def print_warn_log(warn_msg): - """ - Function Description: - print warn log. - Parameter: - warn_msg: the warning message. - """ - _print_log("WARNING", warn_msg) - - -def check_mode_valid(mode): - if mode not in Const.DUMP_MODE: - msg = "Current mode '%s' is not supported. Please use the field in %s" % \ - (mode, Const.DUMP_MODE) - raise CompareException(CompareException.INVALID_DUMP_MODE, msg) - - -def check_file_or_directory_path(path, isdir=False): - """ - Function Description: - check whether the path is valid - Parameter: - path: the path to check - isdir: the path is dir or file - Exception Description: - when invalid data throw exception - """ - if isdir: - if not os.path.exists(path): - print_error_log('The path {} is not exist.'.format(path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - - if not os.path.isdir(path): - print_error_log('The path {} is not a directory.'.format(path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - - if not os.access(path, os.W_OK): - print_error_log( - 'The path {} does not have permission to write. Please check the path permission'.format(path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - else: - if not os.path.isfile(path): - print_error_log('{} is an invalid file or non-exist.'.format(path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - - if not os.access(path, os.R_OK): - print_error_log( - 'The path {} does not have permission to read. Please check the path permission'.format(path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - -def _check_pkl(pkl_file_handle, file_name): - tensor_line = pkl_file_handle.readline() - if len(tensor_line) == 0: - print_error_log("dump file {} have empty line!".format(file_name)) - raise CompareException(CompareException.INVALID_DUMP_FILE) - pkl_file_handle.seek(0, 0) - - -def check_file_mode(npu_pkl, bench_pkl, stack_mode): - npu_pkl_name = os.path.split(npu_pkl)[-1] - bench_pkl_name = os.path.split(bench_pkl)[-1] - - if not npu_pkl_name.startswith("api_stack") and not bench_pkl_name.startswith("api_stack"): - if stack_mode: - print_error_log("The current file does not contain stack information, please turn off the stack_mode") - raise CompareException(CompareException.INVALID_COMPARE_MODE) - elif npu_pkl_name.startswith("api_stack") and bench_pkl_name.startswith("api_stack"): - if not stack_mode: - print_error_log("The current file contains stack information, please turn on the stack_mode") - raise CompareException(CompareException.INVALID_COMPARE_MODE) - else: - print_error_log("The dump mode of the two files is not same, please check the dump files") - raise CompareException(CompareException.INVALID_COMPARE_MODE) - - -def check_file_size(input_file, max_size): - try: - file_size = os.path.getsize(input_file) - except OSError as os_error: - print_error_log('Failed to open "%s". %s' % (input_file, str(os_error))) - raise CompareException(CompareException.INVALID_FILE_ERROR) - if file_size > max_size: - print_error_log('The size (%d) of %s exceeds (%d) bytes, tools not support.' - % (file_size, input_file, max_size)) - raise CompareException(CompareException.INVALID_FILE_ERROR) - - -def get_dump_data_path(dump_dir): - """ - Function Description: - traverse directories and obtain the absolute path of dump data - Parameter: - dump_dir: dump data directory - Return Value: - dump data path,file is exist or file is not exist - """ - dump_data_path = None - file_is_exist = False - - check_file_or_directory_path(dump_dir, True) - for dir_path, sub_paths, files in os.walk(dump_dir): - if len(files) != 0: - dump_data_path = dir_path - file_is_exist = True - break - dump_data_path = dir_path - return dump_data_path, file_is_exist - - -def get_api_name_from_matcher(name): - api_matcher = re.compile(Const.API_PATTERN) - match = api_matcher.match(name) - return match.group(1) if match else "" - - -def modify_dump_path(dump_path, mode): - if mode == Const.ALL: - return dump_path - file_name = os.path.split(dump_path) - mode_file_name = mode + "_" + file_name[-1] - return os.path.join(file_name[0], mode_file_name) - - -def create_directory(dir_path): - """ - Function Description: - creating a directory with specified permissions - Parameter: - dir_path: directory path - Exception Description: - when invalid data throw exception - """ - if not os.path.exists(dir_path): - try: - os.makedirs(dir_path, mode=0o700) - except OSError as ex: - print_error_log( - 'Failed to create {}.Please check the path permission or disk space .{}'.format(dir_path, str(ex))) - raise CompareException(CompareException.INVALID_PATH_ERROR) - - -def execute_command(cmd): - """ - Function Description: - run the following command - Parameter: - cmd: command - Exception Description: - when invalid command throw exception - """ - print_info_log('Execute command:%s' % cmd) - process = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - while process.poll() is None: - line = process.stdout.readline() - line = line.strip() - if line: - print(line) - if process.returncode != 0: - print_error_log('Failed to execute command:%s' % " ".join(cmd)) - raise CompareException(CompareException.INVALID_DATA_ERROR) - - -def save_numpy_data(file_path, data): - """ - save_numpy_data - """ - if not os.path.exists(os.path.dirname(file_path)): - os.makedirs(os.path.dirname(file_path)) - np.save(file_path, data) - - -def parse_arg_value(values): - """ - parse dynamic arg value of atc cmdline - """ - value_list = [] - for item in values.split(Const.SEMICOLON): - value_list.append(parse_value_by_comma(item)) - return value_list - - -def parse_value_by_comma(value): - """ - parse value by comma, like '1,2,4,8' - """ - value_list = [] - value_str_list = value.split(Const.COMMA) - for value_str in value_str_list: - value_str = value_str.strip() - if value_str.isdigit() or value_str == '-1': - value_list.append(int(value_str)) - else: - print_error_log("please check your input shape.") - raise CompareException(CompareException.INVALID_PARAM_ERROR) - return value_list - - -def get_data_len_by_shape(shape): - data_len = 1 - for item in shape: - if item == -1: - print_error_log("please check your input shape, one dim in shape is -1.") - return -1 - data_len = data_len * item - return data_len - - -def add_time_as_suffix(name): - return '{}_{}.csv'.format(name, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) - - -def get_time(): - return datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S") - - -def format_value(value): - return '{:.6f}'.format(value) - - -def torch_device_guard(func): - if is_gpu: - return func - # Parse args/kwargs matched torch.device objects - - @torch_npu_device_guard - def wrapper(*args, **kwargs): - return func(*args, **kwargs) - return wrapper - - -def seed_all(seed=1234, mode=False): - random.seed(seed) - os.environ['PYTHONHASHSEED'] = str(seed) - np.random.seed(seed) - torch.manual_seed(seed) - torch.use_deterministic_algorithms(mode) - if is_gpu: - torch.cuda.manual_seed_all(seed) - torch.cuda.manual_seed(seed) - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.enable = False - torch.backends.cudnn.benchmark = False - else: - torch_npu.npu.manual_seed_all(seed) - torch_npu.npu.manual_seed(seed) - - -def get_process_rank(model): - print_info_log("Rank id is not provided. Trying to get the rank id of the model.") - try: - device = next(model.parameters()).device - except StopIteration: - print_warn_log('There is no parameter in the model. Fail to get rank id.') - return 0, False - if device.type == 'cpu': - print_warn_log("Warning: the debugger is unable to get the rank id. " - "This may cause the dumpped data to be corrupted in the " - "case of distributed training. (You may ignore this if you are using only one card.) " - "Transfer the model to npu or gpu before register_hook() to avoid this warning.") - return 0, False - else: - return device.index, True diff --git a/debug/tools/api_ut_tools/dump/.keep b/debug/tools/api_ut_tools/dump/.keep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/debug/tools/api_ut_tools/dump/dump.py b/debug/tools/api_ut_tools/dump/dump.py deleted file mode 100644 index 97b325a2de6..00000000000 --- a/debug/tools/api_ut_tools/dump/dump.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import inspect -import json -import os -import stat -import numpy as np -import torch -import threading - -try: - import torch_npu -except ImportError: - is_gpu = True -else: - is_gpu = False - -from .utils import DumpUtil, _set_dump_switch4api_list, make_dump_data_dir - -from ..common.utils import print_warn_log, Const, print_info_log, modify_dump_path -from ..dump.utils import check_writable - -forward_init_status = False -backward_init_status = False - -backward_threading_id = 0 - - -class DataInfo(object): - def __init__(self, data, save_data, summary_data, dtype, shape): - self.data = data - self.save_data = save_data - self.summary_data = summary_data - self.dtype = dtype - self.shape = shape - - -def get_not_float_tensor_info(data): - summary_data = [] - if data.numel() == 0 or data.dtype == torch.bool: - tensor_max = [] - tensor_min = [] - tensor_mean = [] - elif len(data.shape) == 0: - tensor_max = data.cpu().detach().float().numpy().tolist() - tensor_min = data.cpu().detach().float().numpy().tolist() - tensor_mean = data.cpu().detach().float().numpy().tolist() - else: - tensor_max = torch._C._VariableFunctionsClass.max(data).cpu().detach().float().numpy().tolist() - tensor_min = torch._C._VariableFunctionsClass.min(data).cpu().detach().float().numpy().tolist() - tensor_mean = torch._C._VariableFunctionsClass.mean(data.float()).cpu().detach().float().numpy().tolist() - saved_tensor = data.contiguous().cpu().detach().numpy() - summary_data.extend([tensor_max, tensor_min, tensor_mean]) - return DataInfo(data, saved_tensor, summary_data, str(data.dtype), tuple(data.shape)) - - -def get_scalar_data_info(data): - summary_data = [data, data, data] - return DataInfo(data, data, summary_data, str(type(data)), str([])) - - -def get_float_tensor_info(data): - summary_data = [] - tensor_max = torch._C._VariableFunctionsClass.max(data).cpu().detach().float().numpy().tolist() - tensor_min = torch._C._VariableFunctionsClass.min(data).cpu().detach().float().numpy().tolist() - tensor_mean = torch._C._VariableFunctionsClass.mean(data).cpu().detach().float().numpy().tolist() - saved_tensor = data.contiguous().cpu().detach().numpy() - summary_data.extend([tensor_max, tensor_min, tensor_mean]) - return DataInfo(data, saved_tensor, summary_data, str(data.dtype), tuple(data.shape)) - - -def dump_tensor(args): - global data_info - args_list = [] - for x in args: - if isinstance(x, torch.Tensor): - if x.numel() == 0 or len(x.shape) == 0 or not x.is_floating_point(): - data_info = get_not_float_tensor_info(x) - else: - data_info = get_float_tensor_info(x) - arg = {"dtype": data_info.dtype, - "shape": data_info.shape, - "type": "torch.Tensor", - "Max": data_info.summary_data[0], - "Min": data_info.summary_data[1]} - else: - arg = {"value": None, - "type": type(x)} - args_list.append(arg) - return args_list - - -def dump_api_tensor(module, name_template, out_feat, dump_file): - api_params_dict = dict() - api_dict = dict() - if Const.BACKWARD in name_template and DumpUtil.dump_mode != Const.FORWARD: - path = os.path.dirname(dump_file) - dump_file = os.path.join(path, "dump_backward.pkl") - api_params_dict["args"] = dump_tensor(out_feat) - elif Const.BACKWARD not in name_template and DumpUtil.dump_mode != Const.BACKWARD: - if module.input_args: - args_list = dump_tensor(module.input_args) - api_params_dict["args"] = args_list - if module.input_kwargs: - api_params_dict["kwargs"] = module.input_kwargs - api_dict[name_template] = api_params_dict - with os.fdopen(os.open(dump_file, os.O_RDWR | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), - "a") as f: - json.dump(api_dict, f) - f.write('\n') - - -def dump_acc_cmp(name, out_feat, module): - dump_file = DumpUtil.get_dump_path() - _set_dump_switch4api_list(name) - - dump_file = modify_dump_path(dump_file, DumpUtil.dump_switch_mode) - - if DumpUtil.get_dump_switch(): - if DumpUtil.dump_init_enable: - DumpUtil.dump_init_enable = False - DumpUtil.dump_data_dir = make_dump_data_dir(dump_file) \ - if DumpUtil.dump_switch_mode not in [Const.STACK, Const.ACL] else "" - if os.path.exists(dump_file) and not os.path.isdir(dump_file): - check_writable(dump_file) - os.remove(dump_file) - - if DumpUtil.dump_switch_mode in [Const.ALL, Const.API_LIST]: - dump_api_tensor(module, name, out_feat, dump_file) - - - - -def acc_cmp_dump(name): - - def acc_cmp_hook(module, in_feat, out_feat): - dump_acc_cmp(name, out_feat, module) - if hasattr(module, "input_args"): - del module.input_args - if hasattr(module, "input_kwargs"): - del module.input_kwargs - - return acc_cmp_hook diff --git a/debug/tools/api_ut_tools/dump/utils.py b/debug/tools/api_ut_tools/dump/utils.py deleted file mode 100644 index bebb693945a..00000000000 --- a/debug/tools/api_ut_tools/dump/utils.py +++ /dev/null @@ -1,219 +0,0 @@ -import os -import shutil -import sys -from pathlib import Path - -from ..common.utils import print_error_log, CompareException, DumpException, Const, get_time, print_info_log, \ - check_mode_valid, get_api_name_from_matcher - -from ..common.version import __version__ - -dump_count = 0 -range_begin_flag, range_end_flag = False, False - - -class DumpUtil(object): - dump_data_dir = None - dump_path = None - dump_switch = None - dump_switch_mode = Const.ALL - dump_switch_scope = [] - dump_init_enable = False - dump_api_list = [] - dump_filter_switch = None - dump_mode = Const.ALL - backward_input = {} - dump_dir_tag = 'ptdbg_dump' - dump_config = None - - @staticmethod - def set_dump_path(save_path): - DumpUtil.dump_path = save_path - DumpUtil.dump_init_enable = True - - @staticmethod - def set_dump_config(dump_config): - DumpUtil.dump_config = dump_config - - @staticmethod - def set_dump_switch(switch, mode, scope, api_list, filter_switch, dump_mode): - DumpUtil.dump_switch = switch - DumpUtil.dump_switch_mode = mode - DumpUtil.dump_init_enable = True - DumpUtil.dump_switch_scope = scope - DumpUtil.dump_api_list = [api.lower() for api in api_list] - DumpUtil.dump_filter_switch = filter_switch - DumpUtil.dump_mode = dump_mode - if mode == Const.ACL: - DumpUtil.dump_switch_scope = [api_name.replace("backward", "forward") for api_name in scope] - - def check_list_or_acl_mode(name_prefix): - global dump_count - for item in DumpUtil.dump_switch_scope: - if name_prefix.startswith(item): - dump_count = dump_count + 1 - return True - - def check_range_mode(name_prefix): - global range_begin_flag - global range_end_flag - if name_prefix.startswith(DumpUtil.dump_switch_scope[0]): - range_begin_flag = True - return True - if name_prefix.startswith(DumpUtil.dump_switch_scope[1]): - range_end_flag = True - return True - if range_begin_flag and not range_end_flag: - return True - return False - - def check_stack_mode(name_prefix): - if len(DumpUtil.dump_switch_scope) == 0: - return True - elif len(DumpUtil.dump_switch_scope) == 1: - return name_prefix.startswith(DumpUtil.dump_switch_scope[0]) - elif len(DumpUtil.dump_switch_scope) == 2: - return DumpUtil.check_range_mode(name_prefix) - else: - print_error_log("dump scope is invalid, Please set the scope mode in" - " set_dump_switch with 'all', 'list', 'range', 'stack', 'acl', 'api_list'!") - return False - - check_mapper = { - Const.LIST: check_list_or_acl_mode, - Const.ACL: check_list_or_acl_mode, - Const.RANGE: check_range_mode, - Const.STACK: check_stack_mode - } - - @staticmethod - def check_switch_scope(name_prefix): - if DumpUtil.dump_switch_mode in DumpUtil.check_mapper: - check_func = DumpUtil.check_mapper[DumpUtil.dump_switch_mode] - return check_func(name_prefix) - return False - - @staticmethod - def get_dump_path(): - if DumpUtil.dump_path: - return DumpUtil.dump_path - - if DumpUtil.dump_switch_mode == Const.ALL: - raise RuntimeError("get_dump_path: the file path is empty," - " you must use set_dump_path to set a valid dump path!!!") - else: - dir_path = os.path.realpath("./") - dump_file_name = "scope_dump_{}_{}_{}.pkl".format( - DumpUtil.dump_switch_mode, DumpUtil.dump_switch_scope[0], get_time()) - DumpUtil.dump_path = os.path.join(dir_path, dump_file_name) - return DumpUtil.dump_path - - @staticmethod - def get_dump_switch(): - return DumpUtil.dump_switch == "ON" - - -def set_dump_path(fpath=None, dump_tag='ptdbg_dump'): - if fpath is None: - raise RuntimeError("set_dump_path '{}' error, please set a valid filename".format(fpath)) - return - real_path = os.path.realpath(fpath) - if not os.path.isdir(real_path): - print_error_log( - "set_dump_path '{}' error, the path is not a directory please set a valid directory.".format(real_path)) - raise DumpException(DumpException.INVALID_PATH_ERROR) - DumpUtil.set_dump_path(real_path) - DumpUtil.dump_dir_tag = dump_tag - - -def generate_dump_path_str(): - if DumpUtil.dump_switch_mode == 'acl': - if DumpUtil.dump_config == '': - print_error_log("Please provide dump config for register hook before turning on dump switch!") - raise DumpException(DumpException.NONE_ERROR) - dump_path = f"according to dump config {DumpUtil.dump_config}" - else: - dump_path = f"to {DumpUtil.dump_path}" - return dump_path - - -def set_dump_switch(switch, mode=Const.ALL, scope=[], api_list=[], filter_switch=Const.ON, dump_mode=Const.ALL): - try: - check_mode_valid(mode) - assert switch in ["ON", "OFF"], "Please set dump switch with 'ON' or 'OFF'." - assert filter_switch in ["ON", "OFF"], "Please set filter_switch with 'ON' or 'OFF'." - assert dump_mode in ["all", "forward", "backward"], "Please set dump_mode with 'all' or 'forward' or 'backward'." - if mode == Const.RANGE: - assert len(scope) == 2, "set_dump_switch, scope param set invalid, it's must be [start, end]." - if mode == Const.LIST: - assert len(scope) != 0, "set_dump_switch, scope param set invalid, it's should not be an empty list." - if mode == Const.STACK: - assert len(scope) <= 2, "set_dump_switch, scope param set invalid, it's must be [start, end] or []." - if mode == Const.ACL: - assert len(scope) == 1, "set_dump_switch, scope param set invalid, only one api name is supported in acl mode." - if mode == Const.API_LIST: - assert isinstance(api_list, list) and len(api_list) >= 1, \ - "Current dump mode is 'api_list', but the content of api_list parameter is empty or valid." - except (CompareException, AssertionError) as err: - print_error_log(str(err)) - sys.exit() - - if switch == "OFF": - dump_path_str = generate_dump_path_str() - DumpUtil.set_dump_switch(switch, mode=mode, scope=scope, api_list=api_list, filter_switch=filter_switch, dump_mode=dump_mode) - if switch == "ON": - dump_path_str = generate_dump_path_str() - - global dump_count - if switch == "ON": - print_info_log(f"Dump switch is turned on. Dump data will be saved {dump_path_str}. ") - if mode == Const.LIST: - dump_count = 0 - else: - print_info_log(f"Dump switch is turned off. Dump data has been saved {dump_path_str}. ") - if mode == Const.LIST: - print_info_log("The number of matched dump is {}".format(dump_count)) - -def _set_dump_switch4api_list(name): - if DumpUtil.dump_api_list: - api_name = get_api_name_from_matcher(name) - DumpUtil.dump_switch = "ON" if api_name in DumpUtil.dump_api_list else "OFF" - - -def set_backward_input(backward_input): - for index, api_name in enumerate(DumpUtil.dump_switch_scope): - DumpUtil.backward_input[api_name] = backward_input[index] - - -def make_dump_data_dir(dump_file_name): - dump_path, file_name = os.path.split(os.path.realpath(dump_file_name)) - name_body, name_extension = os.path.splitext(file_name) - output_dir = os.path.join(dump_path, f"{name_body}") - if not os.path.exists(output_dir): - os.mkdir(output_dir, mode=0o750) - else: - shutil.rmtree(output_dir, ignore_errors=True) - os.mkdir(output_dir, mode=0o750) - return output_dir - - -def make_dump_dirs(rank): - dump_file_name, dump_file_name_body = "dump.pkl", "dump" - dump_root_dir = DumpUtil.dump_path if DumpUtil.dump_path else "./" - tag_dir = os.path.join(dump_root_dir, DumpUtil.dump_dir_tag + f'_v{__version__}') - Path(tag_dir).mkdir(mode=0o750, parents=True, exist_ok=True) - rank_dir = os.path.join(tag_dir, 'rank' + str(rank)) - if not os.path.exists(rank_dir): - os.mkdir(rank_dir, mode=0o750) - DumpUtil.dump_dir = rank_dir - dump_file_path = os.path.join(rank_dir, dump_file_name) - DumpUtil.set_dump_path(dump_file_path) - - -def check_writable(dump_file): - if not os.access(dump_file, os.W_OK): - print_error_log( - 'The path {} does not have permission to write. Please check the path permission'.format( - dump_file)) - raise DumpException(DumpException.INVALID_PATH_ERROR) - diff --git a/debug/tools/api_ut_tools/hook_module/.keep b/debug/tools/api_ut_tools/hook_module/.keep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/debug/tools/api_ut_tools/hook_module/hook_module.py b/debug/tools/api_ut_tools/hook_module/hook_module.py deleted file mode 100644 index c18b37a9c9a..00000000000 --- a/debug/tools/api_ut_tools/hook_module/hook_module.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - - -import functools - -import torch -import torch.nn as nn -import torch.utils.hooks as full_hooks - -module_count = {} -g_stop_hook = False - - -class HOOKModule(nn.Module): - - def __init__(self, hook) -> None: - super(HOOKModule, self).__init__() - self.has_overflow = False - self.input_args = tuple() - self.input_kwargs = dict() - self._enable_hook = True - prefix = "" - if hasattr(self, "prefix_op_name_"): - prefix = self.prefix_op_name_ - - if prefix not in module_count: - module_count[prefix] = 1 - prefix += '0*' - else: - module_count[prefix] += 1 - prefix = prefix + str(module_count[prefix] - 1) + '*' - - self.register_forward_hook(hook(prefix + "forward")) - self.register_backward_hook(hook(prefix + "backward")) - - def __call__(self, *input, **kwargs): - changed = False - global g_stop_hook - if g_stop_hook: - self._enable_hook = False - else: - g_stop_hook = True - changed = True - result = self._call_func(*input, **kwargs) - if changed: - g_stop_hook = False - return result - - def _call_func(self, *input, **kwargs): - if self._enable_hook: - full_backward_hooks, non_full_backward_hooks = [], [] - if len(self._backward_hooks) > 0: - full_backward_hooks, non_full_backward_hooks = self._get_backward_hooks() - for hook in self._forward_pre_hooks.values(): - result = hook(self, input) - if result is not None: - if not isinstance(result, tuple): - result = (result,) - input = result - bw_hook = None - if len(full_backward_hooks) > 0: - bw_hook = full_hooks.BackwardHook(self, full_backward_hooks) - input = bw_hook.setup_input_hook(input) - self.input_args = input - self.input_kwargs = kwargs - if torch._C._get_tracing_state(): - result = self._slow_forward(*input, **kwargs) - else: - result = self.forward(*input, **kwargs) - for hook in self._forward_hooks.values(): - hook_result = hook(self, input, result) - if hook_result is not None: - result = hook_result - if bw_hook: - result = bw_hook.setup_output_hook(result) - if len(non_full_backward_hooks) > 0: - var = result - while not isinstance(var, torch.Tensor): - if isinstance(var, dict): - var = next((v for v in var.values() if isinstance(v, torch.Tensor))) - elif isinstance(var, (list, tuple)): - if var: - var = var[0] - else: - return result - else: - return result - grad_fn = var.grad_fn - if grad_fn is not None: - for hook in non_full_backward_hooks: - wrapper = functools.partial(hook, self) - functools.update_wrapper(wrapper, hook) - grad_fn.register_hook(wrapper) - self._maybe_warn_non_full_backward_hook(input, result, grad_fn) - return result - else: - forward_call = (self._slow_forward if torch._C._get_tracing_state() else self.forward) - return forward_call(*input, **kwargs) diff --git a/debug/tools/api_ut_tools/hook_module/register_hook.py b/debug/tools/api_ut_tools/hook_module/register_hook.py deleted file mode 100644 index 35d02bb641c..00000000000 --- a/debug/tools/api_ut_tools/hook_module/register_hook.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import functools -import os - -import torch - -from . import wrap_torch, wrap_functional, wrap_tensor -from .hook_module import HOOKModule -from ..common.utils import check_file_or_directory_path, print_error_log, CompareException, Const, \ - print_info_log, print_warn_log, get_process_rank -from ..dump.utils import make_dump_dirs -from ..dump.dump import acc_cmp_dump - -try: - import torch_npu -except ImportError: - is_gpu = True -else: - is_gpu = False - -make_dir_flag = True - - -def initialize_hook(hook): - wrap_tensor.wrap_tensor_ops_and_bind(hook) - for attr_name in dir(wrap_tensor.HOOKTensor): - if attr_name.startswith("wrap_"): - setattr(torch.Tensor, attr_name[5:], getattr(wrap_tensor.HOOKTensor, attr_name)) - - wrap_torch.wrap_torch_ops_and_bind(hook) - for attr_name in dir(wrap_torch.HOOKTorchOP): - if attr_name.startswith("wrap_"): - setattr(torch, attr_name[5:], getattr(wrap_torch.HOOKTorchOP, attr_name)) - - wrap_functional.wrap_functional_ops_and_bind(hook) - for attr_name in dir(wrap_functional.HOOKFunctionalOP): - if attr_name.startswith("wrap_"): - setattr(torch.nn.functional, attr_name[5:], getattr(wrap_functional.HOOKFunctionalOP, attr_name)) - - -def register_hook(): - global make_dir_flag - if make_dir_flag: - make_dump_dirs(0) - make_dir_flag = False - initialize_hook(acc_cmp_dump) diff --git a/debug/tools/api_ut_tools/hook_module/support_wrap_ops.yaml b/debug/tools/api_ut_tools/hook_module/support_wrap_ops.yaml deleted file mode 100644 index 6aefa1460b3..00000000000 --- a/debug/tools/api_ut_tools/hook_module/support_wrap_ops.yaml +++ /dev/null @@ -1,1000 +0,0 @@ -# Copyright (c) 2020 Huawei Technologies Co., Ltd -# All rights reserved. -# -# Licensed under the BSD 3-Clause License (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://opensource.org/licenses/BSD-3-Clause -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# List of ops that register hooks - -functional: - - conv1d - - conv2d - - conv3d - - conv_transpose1d - - conv_transpose2d - - conv_transpose3d - - conv_tbc - - avg_pool1d - - avg_pool2d - - avg_pool3d - - fractional_max_pool2d_with_indices - - fractional_max_pool2d - - fractional_max_pool3d_with_indices - - fractional_max_pool3d - - max_pool1d_with_indices - - max_pool1d - - max_pool2d_with_indices - - max_pool2d - - max_pool3d_with_indices - - max_pool3d - - max_unpool1d - - max_unpool2d - - max_unpool3d - - lp_pool2d - - lp_pool1d - - adaptive_max_pool1d_with_indices - - adaptive_max_pool1d - - adaptive_max_pool2d_with_indices - - adaptive_max_pool2d - - adaptive_max_pool3d_with_indices - - adaptive_max_pool3d - - adaptive_avg_pool1d - - adaptive_avg_pool2d - - adaptive_avg_pool3d - - dropout - - alpha_dropout - - dropout2d - - dropout3d - - feature_alpha_dropout - - threshold - - threshold_ - - relu - - relu_ - - glu - - hardtanh - - hardtanh_ - - relu6 - - elu - - elu_ - - selu - - selu_ - - celu - - celu_ - - leaky_relu - - leaky_relu_ - - prelu - - rrelu - - rrelu_ - - logsigmoid - - gelu - - hardshrink - - tanhshrink - - softsign - - softplus - - softmin - - softmax - - gumbel_softmax - - log_softmax - - softshrink - - tanh - - sigmoid - - hardsigmoid - - linear - - bilinear - - silu - - hardswish - - embedding - - embedding_bag - - batch_norm - - instance_norm - - layer_norm - - group_norm - - local_response_norm - - ctc_loss - - nll_loss - - poisson_nll_loss - - gaussian_nll_loss - - kl_div - - cross_entropy - - binary_cross_entropy - - binary_cross_entropy_with_logits - - smooth_l1_loss - - l1_loss - - mse_loss - - margin_ranking_loss - - hinge_embedding_loss - - multilabel_margin_loss - - soft_margin_loss - - multilabel_soft_margin_loss - - cosine_embedding_loss - - multi_margin_loss - - pixel_shuffle - - pixel_unshuffle - - channel_shuffle - - upsample - - interpolate - - upsample_nearest - - upsample_bilinear - - grid_sample - - affine_grid - - pad - - pairwise_distance - - pdist - - cosine_similarity - - one_hot - - triplet_margin_loss - - triplet_margin_with_distance_loss - - normalize - - unfold - - fold - - multi_head_attention_forward - -tensor: - - __add__ - - __and__ - - __bool__ - - __div__ - - __eq__ - - __ge__ - - __gt__ - - __getitem__ - - __iadd__ - - __iand__ - - __idiv__ - - __ifloordiv__ - - __ilshift__ - - __imod__ - - __imul__ - - __ior__ - - __irshift__ - - __isub__ - - __ixor__ - - __lshift__ - - __matmul__ - - __mod__ - - __mul__ - - __nonzero__ - - __or__ - - __radd__ - - __rmul__ - - __rshift__ - - __sub__ - - __truediv__ - - __xor__ - - abs - - abs_ - - absolute - - absolute_ - - acos - - acos_ - - acosh - - acosh_ - - add - - add_ - - addbmm - - addbmm_ - - addcdiv - - addcdiv_ - - addcmul - - addcmul_ - - addmm - - addmm_ - - addmv - - addmv_ - - addr - - addr_ - - align_as - - align_to - - all - - allclose - - amax - - amin - - angle - - any - - arccos - - arccos_ - - arccosh - - arccosh_ - - arcsin - - arcsin_ - - arcsinh - - arcsinh_ - - arctan - - arctan_ - - arctanh - - arctanh_ - - argmax - - argmin - - argsort - - asin - - asin_ - - asinh - - asinh_ - - atan - - atan2 - - atan2_ - - atan_ - - atanh - - atanh_ - - baddbmm - - baddbmm_ - - bernoulli - - bernoulli_ - - bincount - - bitwise_and - - bitwise_and_ - - bitwise_not - - bitwise_not_ - - bitwise_or - - bitwise_or_ - - bitwise_xor - - bitwise_xor_ - - bmm - - broadcast_to - - cauchy_ - - ceil - - ceil_ - - cholesky - - chunk - - clamp - - cholesky_solve - - cholesky_inverse - - clamp_ - - clamp_max - - clamp_max_ - - clip - - clamp_min - - clamp_min_ - - clip_ - - copysign - - copysign_ - - cos - - cos_ - - cosh - - cosh_ - - count_nonzero - - cummax - - cummin - - cumprod - - cumprod_ - - cumsum - - cumsum_ - - deg2rad - - deg2rad_ - - det - - diag - - diag_embed - - diagflat - - diagonal - - diff - - dist - - digamma - - digamma_ - - div - - div_ - - divide - - divide_ - - dot - - eig - - eq - - eq_ - - erf - - equal - - erf_ - - erfc - - erfc_ - - erfinv - - erfinv_ - - exp - - exp2 - - exp2_ - - expm1 - - exp_ - - expm1_ - - exponential_ - - fill_ - - fix - - fill_diagonal_ - - fix_ - - flip - - fliplr - - flatten - - flipud - - float_power - - float_power_ - - floor - - floor_ - - floor_divide - - floor_divide_ - - fmax - - fmin - - fmod - - fmod_ - - frac - - frac_ - - gather - - gcd - - gcd_ - - ge - - ge_ - - geometric_ - - geqrf - - ger - - greater - - greater_ - - gt - - gt_ - - greater_equal - - greater_equal_ - - hardshrink - - heaviside - - heaviside_ - - histc - - hypot - - hypot_ - - igamma - - igamma_ - - igammac - - igammac_ - - index_add - - index_add_ - - inverse - - index_copy - - index_copy_ - - index_fill - - index_fill_ - - index_put - - index_put_ - - inner - - index_select - - isclose - - isfinite - - isinf - - isnan - - isneginf - - isposinf - - isreal - - kron - - kthvalue - - lcm - - lcm_ - - ldexp - - ldexp_ - - le - - le_ - - lerp - - lerp_ - - where - - less - - less_ - - less_equal - - less_equal_ - - lgamma - - lgamma_ - - log - - log10 - - log10_ - - log1p - - log1p_ - - log2 - - log2_ - - log_ - - log_normal_ - - log_softmax - - logcumsumexp - - logdet - - logaddexp - - logaddexp2 - - logical_and - - logical_and_ - - logical_not - - logit - - logical_not_ - - logical_or - - logical_or_ - - logical_xor - - logical_xor_ - - logit_ - - logsumexp - - lstsq - - lt - - lt_ - - lu_solve - - map2_ - - map_ - - masked_fill - - matmul - - masked_fill_ - - masked_scatter - - masked_scatter_ - - masked_select - - matrix_exp - - max - - maximum - - mean - - matrix_power - - median - - min - - minimum - - mm - - mode - - msort - - mul - - mul_ - - multinomial - - multiply - - multiply_ - - mv - - mvlgamma - - mvlgamma_ - - nansum - - narrow - - narrow_copy - - ne - - ne_ - - neg - - neg_ - - negative - - negative_ - - nonzero - - normal_ - - not_equal - - not_equal_ - - permute - - pinverse - - polygamma - - pow - - pow_ - - polygamma_ - - prelu - - prod - - put_ - - rad2deg - - rad2deg_ - - ravel - - real - - reciprocal - - reciprocal_ - - relu - - relu_ - - remainder - - repeat_interleave - - reshape - - remainder_ - - renorm - - renorm_ - - repeat - - reshape_as - - resize_ - - resize_as_ - - roll - - rot90 - - round - - round_ - - rsqrt - - rsqrt_ - - scatter - - scatter_ - - scatter_add - - scatter_add_ - - select - - sgn - - sgn_ - - sigmoid - - sigmoid_ - - sign - - sign_ - - signbit - - sin - - sin_ - - sinc - - sinc_ - - sinh - - sinh_ - - slogdet - - smm - - softmax - - solve - - sort - - split_with_sizes - - sqrt - - sqrt_ - - square - - square_ - - squeeze - - squeeze_ - - sspaddmm - - std - - sub - - sub_ - - sum - - sum_to_size - - svd - - symeig - - t - - t_ - - take - - tan - - tan_ - - tanh - - tanh_ - - tensor_split - - tile - - topk - - transpose - - transpose_ - - triangular_solve - - tril - - tril_ - - triu - - true_divide - - triu_ - - true_divide_ - - trunc - - trunc_ - - type_as - - unbind - - unflatten - - unfold - - unsafe_chunk - - unsqueeze - - unsafe_split - - unsafe_split_with_sizes - - var - - vdot - - unsqueeze_ - - view_as - - xlogy - - xlogy_ - -torch: - - _adaptive_avg_pool2d - - _add_relu - - _add_relu_ - - _aminmax - - _batch_norm_impl_index - - _convolution - - abs - - abs_ - - absolute - - acos - - acos_ - - acosh - - acosh_ - - adaptive_avg_pool1d - - adaptive_max_pool1d - - add - - addbmm - - addcdiv - - addcmul - - addmm - - addmv - - addmv_ - - addr - - amax - - affine_grid_generator - - align_tensors - - all - - alpha_dropout - - amin - - alpha_dropout_ - - angle - - any - - arange - - arccos - - arccos_ - - arccosh - - arccosh_ - - arcsin - - arcsin_ - - arcsinh - - arcsinh_ - - arctan - - arctan_ - - arctanh - - arctanh_ - - argmax - - argmin - - argsort - - asin - - asin_ - - asinh - - asinh_ - - atan - - atan2 - - atan_ - - atanh - - atanh_ - - atleast_1d - - atleast_2d - - atleast_3d - - avg_pool1d - - baddbmm - - bartlett_window - - batch_norm_backward_elemt - - batch_norm_backward_reduce - - batch_norm_elemt - - batch_norm_gather_stats - - batch_norm_gather_stats_with_counts - - bernoulli - - batch_norm_stats - - batch_norm_update_stats - - bilinear - - bincount - - binomial - - binary_cross_entropy_with_logits - - bitwise_and - - bitwise_not - - bitwise_or - - bitwise_xor - - blackman_window - - block_diag - - bmm - - broadcast_tensors - - broadcast_to - - cartesian_prod - - cat - - cdist - - ceil - - ceil_ - - celu - - celu_ - - chain_matmul - - channel_shuffle - - cholesky - - cholesky_inverse - - cholesky_solve - - choose_qparams_optimized - - chunk - - clamp - - clamp_ - - clamp_max - - clamp_max_ - - clamp_min - - clamp_min_ - - clip - - clip_ - - clone - - column_stack - - combinations - - constant_pad_nd - - conv1d - - conv2d - - conv3d - - conv_tbc - - conv_transpose1d - - conv_transpose2d - - conv_transpose3d - - cos - - convolution - - copysign - - cos_ - - cosh - - cosh_ - - cosine_embedding_loss - - cosine_similarity - - count_nonzero - - cross - - ctc_loss - - cummax - - cummin - - cumprod - - cumsum - - deg2rad - - deg2rad_ - - det - - diag - - diag_embed - - diff - - diagflat - - diagonal - - digamma - - dist - - div - - divide - - dot - - dropout - - dropout_ - - dsmm - - dstack - - eig - - einsum - - embedding - - embedding_bag - - embedding_renorm_ - - eq - - equal - - erf - - erf_ - - erfc - - erfc_ - - erfinv - - exp - - exp2 - - exp2_ - - exp_ - - expm1 - - expm1_ - - eye - - feature_dropout - - feature_alpha_dropout - - feature_alpha_dropout_ - - feature_dropout_ - - fix - - fill_ - - fix_ - - flatten - - flip - - fliplr - - flipud - - float_power - - floor - - floor_ - - floor_divide - - fmax - - fmin - - fmod - - frac - - frac_ - - full - - frobenius_norm - - full_like - - gather - - gcd - - gcd_ - - ge - - geqrf - - ger - - greater - - greater_equal - - grid_sampler - - grid_sampler_2d - - group_norm - - grid_sampler_3d - - gru - - gru_cell - - gt - - hamming_window - - hann_window - - hardshrink - - heaviside - - hinge_embedding_loss - - histc - - hsmm - - hspmm - - hstack - - hypot - - igamma - - igammac - - index_add - - index_copy - - inner - - index_fill - - index_put - - index_put_ - - index_select - - instance_norm - - isclose - - isfinite - - isinf - - isnan - - isneginf - - isposinf - - istft - - kaiser_window - - kl_div - - kron - - kthvalue - - layer_norm - - lcm - - lcm_ - - ldexp - - ldexp_ - - le - - lerp - - less - - less_equal - - lgamma - - linspace - - log - - log10 - - log10_ - - log1p - - log1p_ - - log2 - - log2_ - - log_softmax - - log_ - - logaddexp - - logaddexp2 - - logcumsumexp - - logdet - - logical_and - - logical_not - - logical_or - - logical_xor - - logit - - logit_ - - logspace - - logsumexp - - lstm - - lstm_cell - - lstsq - - lt - - lu_solve - - masked_fill - - margin_ranking_loss - - masked_scatter - - masked_select - - matrix_exp - - matmul - - matrix_power - - matrix_rank - - max - - max_pool1d - - max_pool2d - - max_pool1d_with_indices - - max_pool3d - - maximum - - mean - - median - - min - - minimum - - mm - - mode - - moveaxis - - movedim - - msort - - mul - - multinomial - - multiply - - mv - - mvlgamma - - nan_to_num - - nan_to_num_ - - nanmedian - - nansum - - narrow - - native_batch_norm - - native_group_norm - - narrow_copy - - native_layer_norm - - native_norm - - ne - - neg - - negative - - neg_ - - negative_ - - nextafter - - nonzero - - norm_except_dim - - normal - - not_equal - - nuclear_norm - - pairwise_distance - - pdist - - pinverse - - pixel_shuffle - - pixel_unshuffle - - poisson - - poisson_nll_loss - - polar - - polygamma - - pow - - prelu - - prod - - rad2deg - - promote_types - - rad2deg_ - - range - - ravel - - real - - reciprocal - - relu - - reciprocal_ - - relu_ - - remainder - - renorm - - repeat_interleave - - reshape - - resize_as_ - - roll - - rot90 - - round - - round_ - - rrelu - - rrelu_ - - rsqrt - - row_stack - - rsqrt_ - - rsub - - saddmm - - scalar_tensor - - scatter - - select - - scatter_add - - searchsorted - - selu - - selu_ - - sgn - - sigmoid - - sigmoid_ - - sign - - signbit - - sin - - sin_ - - sinc - - sinc_ - - sinh - - sinh_ - - slogdet - - smm - - softmax - - solve - - sort - - sparse_coo_tensor - - square - - split_with_sizes - - spmm - - sqrt - - sqrt_ - - square_ - - squeeze - - sspaddmm - - stack - - std - - std_mean - - sub - - subtract - - sum - - svd - - swapaxes - - swapdims - - symeig - - t - - take - - tan - - tan_ - - tanh - - tanh_ - - tensordot - - tensor_split - - threshold - - threshold_ - - tile - - topk - - transpose - - trapz - - triangular_solve - - tril - - tril_indices - - triplet_margin_loss - - triu - - triu_indices - - true_divide - - trunc - - trunc_ - - unique_consecutive - - xlogy - - unbind - - unique_dim - - unsafe_chunk - - unsafe_split - - vander - - var - - vdot - - unsafe_split_with_sizes - - unsqueeze - - var_mean - - vstack - - where - - xlogy_ diff --git a/debug/tools/api_ut_tools/hook_module/wrap_functional.py b/debug/tools/api_ut_tools/hook_module/wrap_functional.py deleted file mode 100644 index 0eff94c3413..00000000000 --- a/debug/tools/api_ut_tools/hook_module/wrap_functional.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import os - -import torch -import yaml - -from .hook_module import HOOKModule -from ..common.utils import torch_device_guard - -cur_path = os.path.dirname(os.path.realpath(__file__)) -yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with open(yaml_path, 'r') as f: - WrapFunctionalOps = yaml.safe_load(f).get('functional') - -for f in dir(torch.nn.functional): - locals().update({f: getattr(torch.nn.functional, f)}) - - -def get_functional_ops(): - global WrapFunctionalOps - _all_functional_ops = dir(torch.nn.functional) - return set(WrapFunctionalOps) & set(_all_functional_ops) - - -class HOOKFunctionalOP(object): - pass - - -class FunctionalOPTemplate(HOOKModule): - def __init__(self, op_name, hook): - self.op_name_ = op_name - self.prefix_op_name_ = "Functional*" + str(op_name) + "*" - super().__init__(hook) - - @torch_device_guard - def forward(self, *args, **kwargs): - return eval(self.op_name_)(*args, **kwargs) - - -def wrap_functional_op(op_name, hook): - def functional_op_template(*args, **kwargs): - return FunctionalOPTemplate(op_name, hook)(*args, **kwargs) - - return functional_op_template - - -def wrap_functional_ops_and_bind(hook): - _functional_ops = get_functional_ops() - for op_name in _functional_ops: - setattr(HOOKFunctionalOP, "wrap_" + op_name, wrap_functional_op(op_name, hook)) diff --git a/debug/tools/api_ut_tools/hook_module/wrap_tensor.py b/debug/tools/api_ut_tools/hook_module/wrap_tensor.py deleted file mode 100644 index 07fbca8711d..00000000000 --- a/debug/tools/api_ut_tools/hook_module/wrap_tensor.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import os - -import torch -import yaml - -from .hook_module import HOOKModule -from ..common.utils import torch_device_guard - -cur_path = os.path.dirname(os.path.realpath(__file__)) -yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with open(yaml_path, 'r') as f: - WrapTensorOps = yaml.safe_load(f).get('tensor') - - -def get_tensor_ops(): - global WrapTensorOps - _tensor_ops = dir(torch._C._TensorBase) - return set(WrapTensorOps) & set(_tensor_ops) - - -class HOOKTensor(object): - pass - - -class TensorOPTemplate(HOOKModule): - - def __init__(self, op_name, hook): - self.op_name_ = op_name - self.prefix_op_name_ = "Tensor*" + str(op_name) + "*" - super().__init__(hook) - - @torch_device_guard - def forward(self, *args, **kwargs): - return getattr(torch._C._TensorBase, str(self.op_name_))(*args, **kwargs) - - -def wrap_tensor_op(op_name, hook): - - def tensor_op_template(*args, **kwargs): - return TensorOPTemplate(op_name, hook)(*args, **kwargs) - - return tensor_op_template - - -def wrap_tensor_ops_and_bind(hook): - _tensor_ops = get_tensor_ops() - for op_name in _tensor_ops: - setattr(HOOKTensor, "wrap_" + str(op_name), wrap_tensor_op(op_name, hook)) diff --git a/debug/tools/api_ut_tools/hook_module/wrap_torch.py b/debug/tools/api_ut_tools/hook_module/wrap_torch.py deleted file mode 100644 index 23334be233f..00000000000 --- a/debug/tools/api_ut_tools/hook_module/wrap_torch.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2019-2020. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import os - -import torch -import yaml - -from .hook_module import HOOKModule -from ..common.utils import torch_device_guard - -cur_path = os.path.dirname(os.path.realpath(__file__)) -yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with open(yaml_path, 'r') as f: - WrapTorchOps = yaml.safe_load(f).get('torch') - - -def get_torch_ops(): - global WrapTorchOps - _torch_ops = dir(torch._C._VariableFunctionsClass) - return set(WrapTorchOps) & set(_torch_ops) - - -class HOOKTorchOP(object): - pass - - -class TorchOPTemplate(HOOKModule): - - def __init__(self, op_name, hook): - self.op_name_ = op_name - self.prefix_op_name_ = "Torch_" + str(op_name) + "_" - super().__init__(hook) - - def input_param_need_adapt(self): - special_op_list = ["broadcast_tensors"] - for item in special_op_list: - if item in self.op_name_: - return True - return False - - def einsum_adapt(self, *args): - if len(args) < 2: - raise ValueError('einsum(): must specify the equation string and at least one operand, ' - 'or at least one operand and its subscripts list') - equation = None - operands = None - if isinstance(args[0], torch.Tensor): - def parse_subscript(n: int) -> str: - if n == Ellipsis: - return '...' - if n >= 0 and n < 26: - return chr(ord('A') + n) - if n >= 26 and n < 52: - return chr(ord('a') + n - 26) - raise ValueError('einsum(): subscript in subscript list is not within the valid range [0, 52]') - equation = ','.join(''.join(parse_subscript(s) for s in l) for l in args[1::2]) - - if len(args) % 2 == 1: - equation += '->' + ''.join(parse_subscript(s) for s in args[-1]) - operands = args[:-1:2] - else: - operands = args[::2] - else: - equation = args[0] - operands = args[1:] - - if len(operands) == 1 and isinstance(operands[0], (list, tuple)): - _operands = operands[0] - return self.einsum_adapt(equation, *_operands) - return equation, operands - - @torch_device_guard - def forward(self, *args, **kwargs): - if self.input_param_need_adapt(): - return getattr(torch._C._VariableFunctionsClass, str(self.op_name_))(args, **kwargs) - else: - if self.op_name_ == 'einsum': - args = self.einsum_adapt(*args) - return getattr(torch._C._VariableFunctionsClass, str(self.op_name_))(*args, **kwargs) - - -def wrap_torch_op(op_name, hook): - - def torch_op_template(*args, **kwargs): - return TorchOPTemplate(op_name, hook)(*args, **kwargs) - - return torch_op_template - - -def wrap_torch_ops_and_bind(hook): - _torch_ops = get_torch_ops() - for op_name in _torch_ops: - setattr(HOOKTorchOP, "wrap_" + op_name, wrap_torch_op(op_name, hook)) diff --git a/debug/tools/api_ut_tools/run_ut/.keep b/debug/tools/api_ut_tools/run_ut/.keep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/debug/tools/api_ut_tools/run_ut/run_ut.py b/debug/tools/api_ut_tools/run_ut/run_ut.py deleted file mode 100644 index 5f753e57af7..00000000000 --- a/debug/tools/api_ut_tools/run_ut/run_ut.py +++ /dev/null @@ -1,91 +0,0 @@ -import yaml -import os -import json -import torch - -FLOAT_TYPE = ['torch.float32', 'torch.float', 'torch.float64', 'torch.double', 'torch.float16', \ - 'torch.half', 'torch.bfloat16'] - -cur_path = os.path.dirname(os.path.realpath(__file__)) -yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with open(yaml_path, 'r') as f: - WrapFunctionalOps = yaml.safe_load(f).get('functional') - -for f in dir(torch.nn.functional): - locals().update({f: getattr(torch.nn.functional, f)}) - - -def run_ut(): - print("start") - forward_pkl = open("/home/wangchao/torch_test/dump_data_new/npu/ptdbg_dump_v1.0/rank0/dump.pkl") - backward_pkl = open("/home/wangchao/torch_test/dump_data_new/npu/ptdbg_dump_v1.0/rank0/dump_backward.pkl") - forward_content = forward_pkl.readlines() - backward_content = backward_pkl.readlines() - for api_info in forward_content: - api_json = json.loads(api_info) - for key, value in api_json.items(): - [api_type, api_name, index, mode] = key.split("*") - print(api_name) - api_feature = key.rsplit("*", 1)[0] - args, kwargs = generate_input(value.get("args"), api_json.get("kwargs")) - if api_type == "Functional": - out = eval(api_name)(*args, **kwargs) - if api_type == "Tensor": - out = getattr(torch._C._TensorBase, str(api_name))(*args, **kwargs) - for line in backward_content: - if api_feature in line: - api_back_json = json.loads(line) - for params in api_back_json.values(): - grad = nested_generate_input(params.get("args"), True, False) - out.backward(grad) - input_grad = [tensor.grad for tensor in args if isinstance(tensor, torch.Tensor)] - print("forward") - print(out) - print("backward") - print(input_grad) - - -def generate_input(input_args, input_kwargs, need_backward=True, need_convert=False): # 没有考虑dict of tensor - args = [] - kwargs = {} - - for info in input_args: - args.append(nested_generate_input(info, need_backward, need_convert)) - if kwargs: - for key, info in input_kwargs.items(): - kwargs[key] = nested_generate_input(info, need_backward, need_convert) - return args, kwargs - - -def nested_generate_input(info, need_backward, need_convert): - if isinstance(info, list): - result = [] - for i in info: - result.append(nested_generate_input(i, need_backward, need_convert)) - return result - # return list(map(nested_generate_input, info)) - # elif isinstance(input_info, tuple): - # return tuple(map(generate_input, input_info)) - else: - if info['type'] == 'torch.Tensor': - low, high = info['Min'], info['Max'] - data_dtype = info['dtype'] - if data_dtype in FLOAT_TYPE: #应该搞个float类型列表 - if need_convert and data_dtype == "torch.float16": - data_dtype = "torch.float32" - scale = high - low - rand01 = torch.rand(tuple(info['shape']), dtype=eval(data_dtype)) - inpt = rand01 * scale + low - if need_backward: - inpt.requires_grad_(True) - inpt.retain_grad() - elif 'int' in data_dtype or 'long' in data_dtype: # 应该搞个int类型列表, - inpt = torch.randint(int(low), int(high)+1, tuple(info['shape']), dtype=eval(data_dtype)) # high + 1因为右边是开区间 - else: - print(f'Warning: Dtype is not supported: ', info['dtype']) - raise NotImplementedError() - else: - inpt = info['value'] # 遗留问题：需要考虑是否要转换成原本类型 - return inpt - -run_ut() \ No newline at end of file -- Gitee From 464c9e1dd5ea7f2b4f5859913786b957a2ba540b Mon Sep 17 00:00:00 2001 From: wangchao Date: Mon, 24 Jul 2023 09:20:58 +0800 Subject: [PATCH 19/19] api ut tools first commit --- precision/api_ut_tools/common/utils.py | 10 +++++----- precision/api_ut_tools/dump/utils.py | 9 ++++++--- precision/api_ut_tools/run_ut/run_ut.py | 3 ++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/precision/api_ut_tools/common/utils.py b/precision/api_ut_tools/common/utils.py index 39794456099..24959a62950 100644 --- a/precision/api_ut_tools/common/utils.py +++ b/precision/api_ut_tools/common/utils.py @@ -30,11 +30,11 @@ import torch try: import torch_npu except ImportError: - is_gpu = True + IS_GPU = True else: - is_gpu = False + IS_GPU = False -if not is_gpu: +if not IS_GPU: from torch_npu.utils.device_guard import torch_device_guard as torch_npu_device_guard device = collections.namedtuple('device', ['type', 'index']) @@ -414,7 +414,7 @@ def format_value(value): def torch_device_guard(func): - if is_gpu: + if IS_GPU: return func # Parse args/kwargs matched torch.device objects @@ -430,7 +430,7 @@ def seed_all(seed=1234, mode=False): np.random.seed(seed) torch.manual_seed(seed) torch.use_deterministic_algorithms(mode) - if is_gpu: + if IS_GPU: torch.cuda.manual_seed_all(seed) torch.cuda.manual_seed(seed) torch.backends.cudnn.deterministic = True diff --git a/precision/api_ut_tools/dump/utils.py b/precision/api_ut_tools/dump/utils.py index bebb693945a..e5773e097cf 100644 --- a/precision/api_ut_tools/dump/utils.py +++ b/precision/api_ut_tools/dump/utils.py @@ -142,7 +142,8 @@ def set_dump_switch(switch, mode=Const.ALL, scope=[], api_list=[], filter_switch check_mode_valid(mode) assert switch in ["ON", "OFF"], "Please set dump switch with 'ON' or 'OFF'." assert filter_switch in ["ON", "OFF"], "Please set filter_switch with 'ON' or 'OFF'." - assert dump_mode in ["all", "forward", "backward"], "Please set dump_mode with 'all' or 'forward' or 'backward'." + assert dump_mode in ["all", "forward", "backward"], \ + "Please set dump_mode with 'all' or 'forward' or 'backward'." if mode == Const.RANGE: assert len(scope) == 2, "set_dump_switch, scope param set invalid, it's must be [start, end]." if mode == Const.LIST: @@ -150,7 +151,8 @@ def set_dump_switch(switch, mode=Const.ALL, scope=[], api_list=[], filter_switch if mode == Const.STACK: assert len(scope) <= 2, "set_dump_switch, scope param set invalid, it's must be [start, end] or []." if mode == Const.ACL: - assert len(scope) == 1, "set_dump_switch, scope param set invalid, only one api name is supported in acl mode." + assert len(scope) == 1, \ + "set_dump_switch, scope param set invalid, only one api name is supported in acl mode." if mode == Const.API_LIST: assert isinstance(api_list, list) and len(api_list) >= 1, \ "Current dump mode is 'api_list', but the content of api_list parameter is empty or valid." @@ -160,7 +162,8 @@ def set_dump_switch(switch, mode=Const.ALL, scope=[], api_list=[], filter_switch if switch == "OFF": dump_path_str = generate_dump_path_str() - DumpUtil.set_dump_switch(switch, mode=mode, scope=scope, api_list=api_list, filter_switch=filter_switch, dump_mode=dump_mode) + DumpUtil.set_dump_switch(switch, mode=mode, scope=scope, api_list=api_list, + filter_switch=filter_switch, dump_mode=dump_mode) if switch == "ON": dump_path_str = generate_dump_path_str() diff --git a/precision/api_ut_tools/run_ut/run_ut.py b/precision/api_ut_tools/run_ut/run_ut.py index 1c964ac5f31..79e6ed365fc 100644 --- a/precision/api_ut_tools/run_ut/run_ut.py +++ b/precision/api_ut_tools/run_ut/run_ut.py @@ -81,7 +81,8 @@ def nested_generate_input(info, need_backward, need_convert): inpt.requires_grad_(True) inpt.retain_grad() elif 'int' in data_dtype or 'long' in data_dtype: # 应该搞个int类型列表, - inpt = torch.randint(int(low), int(high)+1, tuple(info['shape']), dtype=eval(data_dtype)) # high + 1因为右边是开区间 + inpt = torch.randint(int(low), int(high)+1, tuple(info['shape']), + dtype=eval(data_dtype)) # high + 1因为右边是开区间 else: print(f'Warning: Dtype is not supported: ', info['dtype']) raise NotImplementedError() -- Gitee