From c597c533dc129585f65a27b4e691172a3e39e35e Mon Sep 17 00:00:00 2001 From: pengxiaopeng Date: Wed, 6 Mar 2024 17:47:52 +0800 Subject: [PATCH] improve gradient tool --- debug/accuracy_tools/grad_tool/README.md | 107 ++++++++++++++++-- .../grad_tool/grad_comparator.py | 51 ++++++--- .../accuracy_tools/grad_tool/grad_monitor.py | 70 +++++++++--- .../accuracy_tools/grad_tool/img/image-2.png | Bin 0 -> 8349 bytes .../accuracy_tools/grad_tool/img/image-3.png | Bin 0 -> 20815 bytes .../accuracy_tools/grad_tool/level_adapter.py | 29 +++-- debug/accuracy_tools/grad_tool/utils.py | 26 ++--- 7 files changed, 213 insertions(+), 70 deletions(-) create mode 100644 debug/accuracy_tools/grad_tool/img/image-2.png create mode 100644 debug/accuracy_tools/grad_tool/img/image-3.png diff --git a/debug/accuracy_tools/grad_tool/README.md b/debug/accuracy_tools/grad_tool/README.md index 80de6a5148..78acffbe68 100644 --- a/debug/accuracy_tools/grad_tool/README.md +++ b/debug/accuracy_tools/grad_tool/README.md @@ -1,12 +1,15 @@ # Ascend模型梯度状态监测工具 -Ascend模型梯度状态检测工具能监控一个模型的梯度数据，并且将梯度数据导出。 +训练状态监控工具提供了两种能力 + +1. 将模型的梯度数据导出。这种功能可以将模型权重的梯度值以统计量的形式采集出来，用以分析问题。 +2. 将两份梯度数据进行相似度对比。这种功能可以发现训练过程中问题出现的step，以及抓取反向过程中的问题。 工具支持PyTorch版本：1.11.0/2.0/2.1。 ## 工具特性 -1. 使用便捷，不需要去训练流程里插入代码 +1. 使用便捷，修改处少 2. 可配置多种过滤项 ## 工具安装 @@ -20,11 +23,13 @@ Ascend模型梯度状态检测工具能监控一个模型的梯度数据，并 2. 安装依赖pandas、pyyaml ```bash - pip3 install pandas pyyaml + pip3 install pandas pyyaml tqdm ``` ## 使用方式 +### 梯度数据导出 + 1. 自己写一个配置文件 config.yaml，样例如下： ```python @@ -61,7 +66,8 @@ Ascend模型梯度状态检测工具能监控一个模型的梯度数据，并 |-----| -- | -- | | L0 | ("param_name", "MD5", "max", "min", "norm", "shape") | 无 | | L1 | ("param_name", "MD5", *intervals, "=0", "max", "min", "norm", "shape") | 无 | - | L2 | ("param_name", "MD5", *intervals, "=0", "max", "min", "norm", "shape") | 有 | + | L2 | ("param_name", "MD5", "max", "min", "norm", "shape") | 有 | + | L3 | ("param_name", "MD5", *intervals, "=0", "max", "min", "norm", "shape") | 有 | intervals 就是根据值分布划分出的区间 @@ -71,25 +77,36 @@ Ascend模型梯度状态检测工具能监控一个模型的梯度数据，并 -2. 在训练流程执行之前传入 config.yaml 的路径实例化一个 GradientDumper 对象，并用该对象的 monitor 方法对需要监控的模型挂上钩子。示例代码如下： +2. 在训练流程执行之前传入 config.yaml 的路径实例化一个 GradientDumper 对象。实例代码如下： ```python from grad_tool.grad_monitor import GradientMonitor gm = GradientMonitor("config_path") - gm.monitor(model) ``` +3. 插入代码监控模型，有两种方法，选择其一即可： + + 推荐：在训练流程中，反向执行之后梯度清零之前的位置，调用 gm.save_grad 并将模型作为参数传入 + + ```python + gm.save_grad(model) + ``` + + 另一种：在训练开始前，调用 gm.monitor 并将模型作为参数传入。这种方式目前不稳定。 + + ```python + gm.monitor(model) + ``` -## 输出结果 +### 输出结果 **输出目录结构**（level 为 L2） ```bash {output_path} - └── {timestamp} - ├── rank_{rank_id} - │ ├── grad_summary_{step}.csv - │ ├── step_{step} - │ │ ├── {param_name}.pt + ├── rank_{rank_id} + │ ├── grad_summary_{step}.csv + │ ├── step_{step} + │ │ ├── {param_name}.pt ``` + {timestamp}：梯度工具导出数据的时候会在 output_path 下生成一个时间戳目录，然后在这个时间戳目录下输出结果 + rank_{rank_id}：在分布式场景下，会记录卡的 rank_id。非分布式场景下，如果是 cpu 则记录进程号，如果是 cpu/gpu 则记录卡号 @@ -116,6 +133,60 @@ Ascend模型梯度状态检测工具能监控一个模型的梯度数据，并 | Norm | L2norm值 | | Shape | 形状 | +### 梯度相似度比对 + +会根据所导出的权重，分step比对梯度相似度，输出每个权重的梯度相似度和总的梯度相似度。单个权重的梯度相似度为两份方向数据的重合度，总的梯度相似度为每个权重的梯度相似度按元素个数加权. + +#### 前提条件 + +1. 需要两份以相同配置导出的梯度数据。 +2. 两份数据导出时都需要将 config.yaml 的 level 设置为 L2 或者 L3，因为比对功能需要方向数据。 + +#### 使用方式 + + 1. 单卡比对。新写一个 python 脚本，里面调用 grad_tool.grad_comparator 的 GradComparator.compare 函数，传入的前两个参数分别为梯度数据的 rank 层目录，顺序无所谓，第三个参数为输出目录。如下所示： + + ```python + from grad_tool.grad_comparator import GradComparator + GradComparator.compare("需要对比的rank_id级目录", + "需要对比的rank_id级目录", + "比对结果输出目录") + ``` + + 2. 多卡比对。新写一个 python 脚本，里面调用 grad_tool.grad_comparator 的 GradComparator.compare_distributed 函数，传入的前两个参数分别为梯度数据的 rank 层目录，顺序无所谓，第三个参数为输出目录。如下所示： + + ```python + from grad_tool.grad_comparator import GradComparator + GradComparator.compare_distributed("配置文件里写的输出目录", + "配置文件里写的输出目录", + "比对结果输出目录") + ``` + +### 比对结果 + +**输出目录结构**(多卡比对结果，单卡则没有 rank_{rank_id} 这一级目录) + +```bash +比对结果输出目录 + ├── rank_{rank_id} + │ ├── similarities.csv + │ └── similarities_picture + │ ├── {param_name}.png + │ └── summary_similarities.png +``` + +**similarities.csv示例** + +![Alt text](img/image-2.png) + +这份文件记录了所有权重在每一步的梯度相似度和总的梯度相似度 + +**summary_similarities.png示例** + +![Alt text](img/image-3.png) + +这是梯度相似度随 step 变化的图片 + ## 公开接口 ```python @@ -128,6 +199,16 @@ GradientMonitor.monitor(model) | ----------- | ------------------------------------------------------------ | -------- | | model | 设置需要监测的模型 | 是 | +```python +GradientMonitor.save_grad(model) +``` + +**参数说明** + +| 参数名称 | 说明 | 是否必选 | +| ----------- | ------------------------------------------------------------ | -------- | +| model | 设置需要监测的模型 | 是 | + ```python GradientMonitor.__init__(config_path) ``` @@ -138,5 +219,7 @@ GradientMonitor.__init__(config_path) | ----------- | ------------------------------------------------------------ | -------- | | config_path | 配置文件路径，需要以.yaml结尾 | 是 | + + # FAQ diff --git a/debug/accuracy_tools/grad_tool/grad_comparator.py b/debug/accuracy_tools/grad_tool/grad_comparator.py index 58a0da2e96..ff6302d4fc 100644 --- a/debug/accuracy_tools/grad_tool/grad_comparator.py +++ b/debug/accuracy_tools/grad_tool/grad_comparator.py @@ -2,18 +2,32 @@ import os import torch from tqdm import tqdm import matplotlib.pyplot as plt -from grad_tool.utils import write_csv, path_check, print_info_log +from grad_tool.utils import write_csv, path_check, print_info_log, create_directory class GradComparator: + @staticmethod + def compare_distributed(path1: str, path2: str, output_dir): + ranks = GradComparator._get_matched_dirs(path1, path2, "rank") + print_info_log(f"the following ranks will be compared: {ranks}") + if not ranks: + raise Exception("no matched ranks for comparison, please dump data in same configuration") + if not os.path.isdir(output_dir): + create_directory(output_dir) + for rank in tqdm(ranks, desc="rank"): + print_info_log(f"now comparing rank {rank}:") + GradComparator.compare(os.path.join(path1, f"rank_{rank}"), + os.path.join(path2, f"rank_{rank}"), + os.path.join(output_dir, f"rank_{rank}")) + @staticmethod def compare(path1: str, path2: str, output_dir): - steps = GradComparator._get_matched_steps(path1, path2) + steps = GradComparator._get_matched_dirs(path1, path2, "step") if not steps: raise Exception("no matched steps for comparison, please dump data in same configuration") similarities = GradComparator._calculate_separated_similarities(path1, path2, steps) if not os.path.isdir(output_dir): - os.makedirs(output_dir) + create_directory(output_dir) GradComparator._save_similarities(similarities, steps, output_dir) @staticmethod @@ -30,12 +44,13 @@ class GradComparator: same_count, total_count = GradComparator._calculate_similarity(pt1, pt2) same_count_summary += same_count total_count_summary += total_count - if pt_file not in similarities: - similarities[pt_file] = [] + param_name = pt_file[:-3] + if param_name not in similarities: + similarities[param_name] = [] if total_count == 0: - similarities[pt_file].append(0) + similarities[param_name].append(0) else: - similarities[pt_file].append(same_count / total_count) + similarities[param_name].append(same_count / total_count) if "summary" not in similarities: similarities["summary"] = [] if total_count_summary == 0: @@ -45,21 +60,21 @@ class GradComparator: return similarities @staticmethod - def _get_matched_steps(path1: str, path2: str): + def _get_matched_dirs(path1: str, path2: str, dir_prefix): path_check(path1, isdir=True) path_check(path2, isdir=True) - steps = [] + dirs = [] for dirname in os.listdir(path1): splits = dirname.split('_') - if not splits or splits[0] != 'step' or not splits[1].isdigit(): + if not splits or splits[0] != dir_prefix or not splits[1].isdigit(): continue folder2 = os.path.join(path2, dirname) - if not os.path.exists(folder2): + if not os.path.isdir(folder2): continue - steps.append(int(splits[1])) - steps = sorted(steps) - return steps + dirs.append(int(splits[1])) + dirs = sorted(dirs) + return dirs @staticmethod def _get_matched_pt_files(path1: str, path2: str, step: int): @@ -89,11 +104,13 @@ class GradComparator: plt.xlabel('steps') plt.ylabel('similarities') plt.title(f'{key}_similarities') - plt.savefig(f'{output_dir}/{key}_similarities.png') - plt.savefig(os.path.join(output_dir, f"{key}_similarities.png")) + picture_dir = os.path.join(output_dir, "similarities_picture") + if not os.path.isdir(picture_dir): + create_directory(picture_dir) + plt.savefig(os.path.join(picture_dir, f"{key}_similarities.png")) plt.close() head_tuple = tuple(['step'] + [str(step) for step in steps]) - write_csv(os.path.join(output_dir, f"{key}_similarities.csv"), [['similarity'] + value], head_tuple) + write_csv(os.path.join(output_dir, "similarities.csv"), [[key] + value], head_tuple) @staticmethod def _calculate_similarity(pt_file1: str, pt_file2: str): diff --git a/debug/accuracy_tools/grad_tool/grad_monitor.py b/debug/accuracy_tools/grad_tool/grad_monitor.py index 1abbc63bc4..cb6ab15bf6 100644 --- a/debug/accuracy_tools/grad_tool/grad_monitor.py +++ b/debug/accuracy_tools/grad_tool/grad_monitor.py @@ -1,10 +1,9 @@ import os import torch - from grad_tool.level_adapter import Level, LevelAdapter from grad_tool.grad_stat_csv import GradStatCsv from grad_tool.utils import get_config, check_numeral_list_ascend, ListCache, data_in_list_target,\ - write_csv, make_localtime_dir, get_rank_id + write_csv, get_rank_id, print_info_log, create_directory, print_warn_log class GradientMonitor: @@ -15,41 +14,53 @@ class GradientMonitor: self._level_adp: Level = LevelAdapter.level_adapter(config.get("level")) self._param_list = config.get('param_list') self._target_ranks = config.get("rank") + print_info_log(f"target rank {self._target_ranks}") self._target_step = config.get("step") + print_info_log(f"target step {self._target_step}") self._bounds = config.get("bounds") if not self._bounds or len(self._bounds) == 0: self._bounds = GradientMonitor.default_bounds check_numeral_list_ascend(self._bounds) - self._output_path = make_localtime_dir(config.get("output_path")) + self._output_path = config.get("output_path") + if not os.path.isdir(self._output_path): + create_directory(self._output_path) + else: + print_warn_log(f"the file in {self._output_path} will be recoverd") self._step = -1 self._list_cache = ListCache() @staticmethod - def hook_fun(param_name, f): + def _hook_fun(param_name, f): def backward_hook(grad): f(param_name, grad) return backward_hook - def model_backward_hook(self, module, gin, gout): + def _rank_in_targets(self): + if not hasattr(self, "_rank"): + raise AttributeError("grad monitor need attribute {_rank}") + return not torch.distributed.is_initialized() or data_in_list_target(getattr(self, "_rank"), self._target_ranks) + + def _model_backward_hook(self, module, gin, gout): + self._step += 1 if not hasattr(self, "_rank"): setattr(self, "_rank", get_rank_id(gout)) - if torch.distributed.is_initialized() and not data_in_list_target(getattr(self, "_rank"), self._target_ranks): + print_info_log(f"rank_{self._rank} exists") + if not self._rank_in_targets(): return self._list_cache.flush() - self._step += 1 if not data_in_list_target(self._step, self._target_step): return - output_path = f'{self._output_path}/rank_{self._rank}/grad_summary_{self._step}.csv' + print_info_log(f"result generate: rank_{self._rank} step_{self._step}") + output_path = os.path.join(self._output_path, f"rank_{getattr(self, '_rank')}", f"grad_summary_{self._step}.csv") write_csv(output_path, [], GradStatCsv.generate_csv_header(level=self._level_adp, bounds=self._bounds)) self._list_cache.set_output_file(output_path) - def save_grad_stat(self, param_name, grad): - if not hasattr(self, "_rank"): - raise AttributeError("grad monitor need attribute {_rank} when save grad stat") - if torch.distributed.is_initialized() and not data_in_list_target(getattr(self, "_rank"), self._target_ranks): + def _save_grad_stat(self, param_name, grad): + if not self._rank_in_targets(): return if not data_in_list_target(self._step, self._target_step): return + print_info_log(f"param result: rank{self._rank} step{self._step} {param_name}") grad_info = GradStatCsv.generate_csv_line( level=self._level_adp, param_name=param_name, @@ -58,10 +69,39 @@ class GradientMonitor: self._list_cache.append(grad_info) self._level_adp.save_grad_direction(param_name, grad, f'{self._output_path}/rank_{self._rank}/step_{self._step}') - def monitor(self, model): - model.register_full_backward_hook(self.model_backward_hook) + last_module = None + for name, module in model.named_modules(): + last_module = module + last_module.register_backward_hook(self._model_backward_hook) for param_name, param in model.named_parameters(): if not data_in_list_target(param_name, self._param_list): continue - param.register_hook(GradientMonitor.hook_fun(param_name, self.save_grad_stat)) \ No newline at end of file + if param is None or param.requires_grad == False: + continue + param.register_hook(GradientMonitor._hook_fun(param_name, self._save_grad_stat)) + + def save_grad(self, model): + self._step += 1 + if not hasattr(self, "_rank"): + setattr(self, "_rank", get_rank_id(next(model.parameters()))) + if not self._rank_in_targets(): + return + if not data_in_list_target(self._step, self._target_step): + return + print_info_log(f"save grad rank_{getattr(self, '_rank')} step_{self._step}") + output_path = os.path.join(self._output_path, f"rank_{getattr(self, '_rank')}", f"grad_summary_{self._step}.csv") + write_csv(output_path, [], GradStatCsv.generate_csv_header(level=self._level_adp, bounds=self._bounds)) + self._list_cache.set_output_file(output_path) + for param_name, param in model.named_parameters(): + if not data_in_list_target(param_name, self._param_list): + continue + if param.grad is not None: + grad = param.grad + elif param.main_grad is not None: + grad = param.main_grad + else: + continue + self._save_grad_stat(param_name, grad) + print_info_log(f"{param_name} is saved") + self._list_cache.flush() diff --git a/debug/accuracy_tools/grad_tool/img/image-2.png b/debug/accuracy_tools/grad_tool/img/image-2.png new file mode 100644 index 0000000000000000000000000000000000000000..75b6a8e073c52f439602e89b4871813b76ff6357 GIT binary patch literal 8349 zcmZ{qcRbsD_x~mK9(>^0um}plq&A*<}(Mi%-T`|5D4p?`s&zP^_Ym7B|4PegF3;W3(xDi<4iC)4>h3P*Tm(&bqCq^D6!%p`WC9er(A?5SjUEvqPSBMc*D?!UAu7tV={-Q;U% z$tUb^0pAA_ID=hn%Ni*>jQ&IyIxHRmuKVe|ibBPrn%=x&W4=MULCSCt-uvp-x*0tu z1blz9VUNGlsT*_I94NX}Te`(b0TN3C1;VbDncy$u;mq%w0sOizF70Jsi3M*>iG0Yc z5@*T1O4euNt>_Epvxrrp-s5fn(ClyoGX4VEC3bUL zI4!wg#0Lf563ap}v*a@G1lQ%J9!5~l#juoXSq=tFOCBGtmZt*S3q@9ASaKr*BPo*$ zSvi@0bM!=Qqx6LXSwdsv#Q+%}hAmek?7k6B?mWDU;s;WffbqSt>SO?blBgcy6`wV} zIoUkB!nJl9fIHUa)u>rNk2Aqr=R5rYy?-4gXi9=;2xcv&99=If6)3W~wMdWkY2TkV zXndb1EJz#?E<8_U0<@$%%+AkUle~B9-Y46{3Ie~NOZ~knu$M@T(Bf1*!ZBH>TZ1Kc zr6MgtTE2N?-h#}9{Lm7Q)bCx2kBEr6xG%NzWn5~Uy3UL}K6%0PStDdJ??b;RQPIhn z@l$owRb)9xJ>z5ATjc5!-t3x`EZ4lDu7r@9n9;U#HPsEGuRJYLK8x+C%$<&~C`JH? z?Vs(*Rfe9_2zX_5Itqa8_8b@Lx2vcjj!H)p((mIV?{L^A2gIZFURavM5L#QiR)!Ve zY)hlr9{lP7U0;T}7kUjECs=Y-l=))e%+<-vo%~olR}Y7V?!|*Y4PrMr!h4>n%!(Z@ z?3nJP(+*b?%VCdm59;OYtUjo-!jD?<@p`6vnaCGsl4i;br`qPEjwffuW}~z+VzjHUC&Fmy`4Z6j1c4xZtp|ekqB#z9j|2QP7DBNZ<|Iw%<$vAOtIe_AU0aws#PI~=O=qd&Orf0)JGl9g>W%g00ar(U z^_5Yf)7%xo47@1lDH2W|jUzs#^CGpUq=;=5FcHIBzLmMq-mIqmR*&rHH@VQ_LZUue zhb6a}y#d+LDx@l9-Yipe6d*kq*P>N!yJ&p$97j?7Ibzd<1o1{IcrdXBGC?cuSk}Rd zG4$9W5<5UMT_PtX{;;8rGx_dnXcnp%>}wF{f^RiKU5akd^J^RJXl0~j8Vo&*lsbvM z&P6-AJ2`X!T^QYKeDBhP)|MbGpJ;9IXmcz{F>o+&L{p1t2?aBgeD_+HFl|YdEKp;P zHNoFX5$fdY3cdwJRJi^ z7<`N6_g;STwIi8U>kJ7bs zBk8zM^dvgnvzzk_3ZxE0?lh5@p>)jKq7_X@1mZEe;E;~8kvD-y;3%{#*kIh?po?O_ z5E;A^m1quNjK}1eCE?0#?Dc#HuE{b7v=hkdDy>!oz(^DS2>fyuay-j}74-HqbZnE~ zo{bEfG0>bS)=MP&wB{(be$hm-MViJHqgkzO&7 zAorqL;XW-}68h{$bCwhttlx#ZJ4)kJf@E>8@C#WTZqJly-bBu94~_r;^niG=-2Z2O z&-glos$N9oJ)zsrqw_0bqUCDaxOGdP*#DYR475;Yi*#0*Mc~NwHh`TW^dfE%wsUBI zlA&8`ztv4_&&9LQBpD^2Rvo*$mU{! z44=OGxDTm6K76Tj@v!5c+b^(ZyzcVY!m`V?Vy}N|vtaaZkNL)kXjOsAjr_H8T`%$v zf4@(C5z!z; zJ@RpzG9Zg_AlzUU+6Vi0f;aJ)8g+yyI06>W28E>xF*a+f@?#I+tq0AAZ^ce?VcuAA zqvKtX3C^xxn=g4A`a1Iq~*O^c`75<0B_5^cB%)u0P6I3Pzz*BEjrp0-tp{y9vRsIRGwzHhl zQ!DFy^fg`RFnhNS>e~oQ@Q@&^jpVF=u2@SezO7(6j+0zovz=Hu7EE=?NBgq**zX)u zI*eu$AJoYk63Q2M1FB{C*QD!fcas@(gMcM+o%NGqh1EsF(O$p4?9AF+zvoYasA9*a zzCbeV?g_Q+8W|~b)ZM7YOl*psT;IH+ksrHRMY%c2G^zHNg1-Tlo*_;7M9&z{4!bXO z`RnTJ5_{hAoEaC$+y?m(YEpp-DY;w^yggc4v4okzj)p;0f;prv4Idfvki(*TQZc#x zhnElb^0>`QEPgGu;YhfEtLtA!LZOv4v&d-wXzW*W>~y+>48SIjZm!_r{mExHl&U|> zVd|T{&&fMXR?uf1mSxzUYMC-xKYo7UL4o9MCVLC}_z~H?BXPQePu#0Dz{t&%ES?f@ z5!;~&M=)X?*@XWjCGu35Wo#-ZNRP0x)VE@R8Reg1?MXMXALUmE#`WB&2id(-F%SAJzJfDGYhs}MR~Vy8M}Vw5WbPCQ_+E>7AG z!%?3!e4b&ugY1z?pce@HVPOzF6LRAe+iz{$*Q+#sHDoL7t_&-yj?HM(@|dbrp61@5 zlgCs8jt@4BlZChHBB9Ye9o&0aC^EY4By`H$=qa6o4JxW@5Zx_rn)mA&WcQh0;@jL8 zB9v^TZk3_RC5Z8Bgq=$es4Z=Ao)V;Cy41Znyd1}heAE9mz&uJKB|v`ht+!a_CKC8) zdX6WtkNED31?SWpGnRf(AhdN&^1HUQ^~yt=;Z5v zRv18MkdAn@HJ!hI?M{h7CZeuU?9{&VsNW@~r3a+jg0T{Xy)RA&^)m~?8&^uZZ|Esj>o^AH|XNz6}MIXU63V#%^jU9`Jp*Eu{tB)A4)HWenM4WAP%>Y^a zvHWEkPI=OHO9!8!$odQy$OogABn2G;3Ta4~LtWc1)(H9y+Wi6$t)rww1!cIQR>McAM`?iUIJRvq8ct@FxTzob=SD(@`lOW$au6c0Kf_Zk-GC6m^4ES=T zSxnjT)3HQP_kxGouMia}+IO5y4GbM3iw&m-s=;A+&T9zQ6DEs{G5Innc)QRO?5s*< z=bRa#UgX9p#N*p>@{2w&IEKcV}uALBSVtM>N9EklcM^}cba+175?WC&JEl%BXi zt{(j{>zb7RXx3a5h@HUekj!+nIfYH}mt=K=*Y2Pj<77>13QyF?t8b@|K$orhf04DBQmZ1X%k}dz6kTw509Xy z@DW|2!ZpC@46bI*xy7{d{%7hpeYz4Y|xO%R6kPDVoRiqzeq#ZBE8* zB~2Tzm3}n^wwOl@T$`qe>-vB!UA&%Qh7!wWUl7??`>kI? zZcR!5uKVh`n@2Sxv}`On$r{Zwz}P(SrFkHeCA+buT6SYZfNCM?DYDs;Sue@6I+)m+ znyKr+AJ7zF{Hk>q(VXHrG$v5Sy82R=N2J3Hm_M?~N;Kqp7bf&M-{)x5)hFV?!4Uc3 zay=Z6xQ`U@f1Z~uTkAM0O!23OWvCblR^N6vyo}!hhFrWqSfmZheI`3spg2ARd?Bkc z-saw{NU*pt+S|nT);ZMHu7@RIvM<51!b2=gvY|jCq$Bk}@t{h>oA7Al!r@LdPS$-=5Cf$-laos?g=h5dM8Emj}_5Umgym)<6Mm z{5>c>i<4`Q!uN2`l7>b z!S+$)2)ZqQ3Gq-#XQaI{LWE-T(pJ7hCti!e+QDFfzKr6Zbl#M%BL&NG*`f1VlrFsN zmDd$fZtH1uZ4O(MgDj=`OGvh?>-P_?lJ_FF)vVhQz9m_;W?2}g|=)`EwCSXN0vle&OsQwD)H)+VIB0=6|IOjGbXwNz|mMgdAasG2l6_u$$*&LemvAD>Q5p zK>0c4Ul<|@e=j8a53aG&i`th~USQ-*xER)_0PkdRkWFrF=+rYcLzj_Q7Zya@6KSlY zHu_ork5&n&eUpQkmJCmDmu>C48frw#-o17%oZtyxfPTi5e^hczveUmvXDvZ?>DZy{ zFkPuud!jvRv7B?Zg?G8BVc>A`KvBBB;6jG|ikw?my2zIu5ZYo^&(5Ql;j)%f5VOv; z=1|JTvb&zC1wy*_r~hPATju~lN=lrUR(1$sTo7r(>Xu%>cFp0^ znwGy`Q1|J=8$N||zE-0tbz=>$3)a5PDDX)JMU2}vMaVf!>isZ3kd9%_opZ};#Mk!P zHl3zd64Jn_yS`lZJ9R-T@>dJG$5@cHiREn-0(07cq07%ig9fy@gK8EW<>1<#aPH`QxGl* z*eo$CD0+8OnR9f*)zjm0tg}Vc%Ty7XoYU8)ng#*ao)OS{kopU+v02x`^uzOv{?oEw zL!U9z>hparkCPzL1k{+f9pg*~Z*&^*28pI`YfxMwIcYKnA(({#Zjm~ZjaS0pVLz|& zW9~j1Dop{>)@o9 zF$W-|;B=(V=>Lc`G>t3lH{^9IWH!qFaRw#L3vWv4e?y5mONq|)Ma95k+r3uM*;4D0 zOerIf3CILd7vOq@FT#rzE3WyKCJ~R@{{>m9xs`=8O=%3elU~lV=V-Ay&#L&QBmWO$ zX}GUxL;z`EXyGQ|ew|1C`RqdmK6v^ut0-d`(a3p;`|P2mExYx{|AtA$K$_m8$`k3N z^KZRpW~{+=x@YHcScg#|criG7k|yvaC& zVy!Ubs_mqwg;3k+Jc{&^gl8*$I#XSKOq|jbjmF$=xswLNMlgHbbR|+c9>sfN8k_e4 z-x%|kXTv(EN2#M9edznwB`*=~be&81+Wwrm{_43%uSrhDZg*|;ih(@!7ujr_1NqMA zJs&h{cJ+ZxiF*#gYp0NaZ`NS2Zql%i%@_M%b|tX>@GwSSWnt(jBb*! zEPe$9&pA57AGNu~L4y#xjjRHa;hc|U36FYg%Xjq9TyX~>{O9I3oPfgZiY=&3L&a+c zkxW=Br9eDjJ?6N8N)#t-MVJtW=AE~d>mcY z>VuKz9HiPsR&V8@q<~RXo9_p-GjWl@pknQNH$L<%Bcq)i_aZd={R=T>%YCbH*1uI9 zvx#cP$hq()t2+vMspV~-joh8hc~f>0{e13sj2V1KUOJsB7@SL*FI+*F#HK&rFrHNubSyh8M;zO){w5?HNDGY5eIkMreou4QszY99>a_4F;gwi-DHM#$teskTv znn~rhpS5>rVjie}pOtW9W-;oclX=rqxd&!=S=Jss*$gYLK=g*FkmDWG@pks8?n4)e zn&}PyWDY9OuTI2ZAHgd2W_J<{*m{Kk*A@8;9K2VD+kkgOCb|G z*J5L9Ld?!uy6Y$!|BSo~>je=qcL=b4&q9gS)m;Rs{u1^Aanb+dtulq9JL1aa07`b0 zDOPm*&y=B?)G1d+qAUS%Y-)J5uVa^kNBt9}UOLBTC$a9bex2CIRJ4w@>#Ar=!qbqN zpE7*e5FthrW*ddVDjPe-iM@U~E5MVNMm(t69Z#;f-=fZ-ktPRND&>T?LETFjB9LT- z7WMz?#%EW3Ksoq+KK^>tp8uREAL*-ZvaW}LQZP9+JJVH;Oo-S4@^;Cq`JltCev@vC2z-D^m3`N<=$+6jofknii#yHW3;NGP{_t%m5( z%ZL~ohG^@#0j(oZBema}3T~v-m7+Pjhz#ievA7uKyDZEac$am6oF#zafSEx*269r$ zIAE!caZ2Do*RrmWM2-f7wM;uFS9XRgFlaYs{ejE-P1h0kfJl2l z*0{IXgD{JNwJkJWXSd*^#6PefAv#U9TcPG-`b4g@aVb`tne}LbWCW~5Pf@-ww?vrd zQzm9^Hsm8CGbB8c>B(gN>(Wci0gl?=C;jucIBNNpMiO@Vj{WGd^eublGeG*yu zJgRR|L+RzOo2uzqdqJKo3gKjY&2!IYdjOuj2Arf{G+sSF{&u}?SLuKGgm*J5>0~lY zOWWCp2L(OHAa~8P&URY?1P(hMeoU24D~;xy%#@g(xG(9dI#aT4zz2s$xTI#03_zM; zE{KeAen(XT5N&8tt-A5oQ|^C!=&*N*-Z-v<>9l}nrZpr~T}?V8 zliL^;$fF?Mv)T$gn_*+zt?v3*ol#9`kllB7FsYY^cy%XaAd-K18sxqzp_Me%r=;-L2&i)I4FN^m7 z@8leFB3rd9%TDuWL=Sn3T7aqqq@4lBe?*v?W`F|R=0jG~*;r~`4<3IG+eN*{9Sxv~ hFC*I^%7zf}l)lWXOEiqz7(uf^XJuw{rOX5#_dk?PTj>A* literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/grad_tool/img/image-3.png b/debug/accuracy_tools/grad_tool/img/image-3.png new file mode 100644 index 0000000000000000000000000000000000000000..92a91f110aba9448df053471a1a0431588ce8648 GIT binary patch literal 20815 zcmdSBbySr>+dc|NqktfYG)Q+yBaJjjcPP@`A-P3Dx;rGK8>G9tOTvwG2pf<_IJ14< z?>pyr{yP7jwQ%Wj@0n-jnVEa8`?{{1u(!%GPcVrw5fBib$jM5nAs`^Of{)TXqggoc(@+J3FoitS)VMKqCB|9i@1lQ@zfUJi86l3+dznFW1wb2BtMJG(Eo z-+^4z&w$OSwTR6!3`$E!H$0Umqbabov~+qp!meOuR?tLkF_D#6s$H4(|LjEWsTdZL z@x@h*lB{gZpFghR2h@y=a+p|HV+mZ`+$a<@q%!R!eD;Hh%-R|X`OVFGr>Cd0_vZ%- zHg$?`wfy|rnedGlx3;#_^z=$DoQ!SL)l^kuY3S({e0*A&p1W}t%+6|PVh_t0I{B+- zDtb*%Pv__4bk==bHWqex^Cq~2em{!Kv8=32arOCgXcmJgAD=eHK2Mm5*VTFmilK|k_GoOwnzw@6dgKf4{tS=#GF>M2{po5`8Qo^r;_H># z2@z)B+Y8N0G9K#$QbAW)BcBs#Mn*Cg-;A&`?HZ?U>(q7qd z-`oo8#+uHP!3hfwS6?6#_I&GiQJXzf^{xlmfY)}BGRZK_a`o^1Wry}oe+9REnGm^EyoZH|4i>8y&fd3ro#ne7%x_bl^hL3I(zuem|4H+~a_DZG zuS1m;FQeh^J@%6)i4XV34_d4w83(H^4S437lQ<8qyOZLRza%j+G2^t#^-@P_MlF5> zqbsJp8mhCow_)`Z{kx}Pc5%2=Vix3ro9*$_SNnI~x$95g%T)buZW!cxK}kbBJ(AlK z*_GIVXh+q)x9%>43SB*~IkGI43as#i>!5FKrKKo6HFN`5jk%?8v4cWO0QL z zSGr5)t9pQ3J@?Oq|9$P7WI8=QI?lkHubSww;y+qEDuu5%1g%rz!A`9<+16yt$es)$H2l;Uyw}be8ehC6VdV& z>*)d|m^wJ|>X%}CtbWgOZf9rHcrnQP*A@TGTHAG1Vsi4g?M)|}tR*cPD(Y|ymE>ka z$KP_O45IXeRI2}Pr2vS9LpMvu5;CRdH;gB^46f1=oq}%r`R#92Ir;~iUH8-^VQZcn zPtvi->2z}v-xKdZ81yjLTqa(7q(i?R;FMQ};nT}k8nqE`nk-3&CjIz?BU02XX2;bOHU#|3IvSZHWWBne;1{CvYtztU#HsqSjq^(9FdOgidFWftwS4>|GS zD*Az=0cMRaEc}N)+b6eKZi>2$T^wEH;m&2wx8>ud{oF4|RbKeyJS4=M#kk$zsQG}c z4{tSUSKbbcaGw6uVXalS-fzm98-skfzc=II@6QL#iGU=s-e;!qS56QMgDR6i=nKyA zg|?3#E(a|lBEm9A-zmkqYEh(<-aSW3kqdnha;4TwG!y$$X4ZGIEtGw)!mwpHj#lAA zf5eL#$8`xe>{YXwZ%W(^u&QV_74=Kf{(i?yt0qKq%-n)NWT(BQ+QR)eeIm{LRJKKE zctL2~qMW5qSCcL4Q_U%Ug$vWrNNv^Lp)E)hk?fC49iVXwQO$cAxn8Fpk>KO!U8u6{ zhq8^h?2H#Lo7m#Ham_vIjHaqC%> zLA%eA&l5h`&NYBxF6c&2SJM#s+i(8tnUZt6+7S78;yH4G$Qb{mbD1O=!QAlubokaE zzq(1~EMc!@G*+$|DUuDm_4wET7jWP*MzTHlI0ENNwM)%#lj2n;&qM9?4siMz1oWpm z*&k6TVF3ulI@&4!AFQ*O9`F?fXN`sf7G*qs7lZf{vHg(`n(ZyN$#yxjTOWxYtg7+C z!T**-^h2jdz%jy4M4{G4DS?Ob+;&e`J@p&x6Ty{J{K7eCx$ajF7a^ zsHVdpiM7V}(?{LZE~f1^L|5b=fWty8Ba^Os95m{jDAnacpirF!M`Jlp!35v*i)k)) z&x}%*VFS?j4*f0c6*n!43iqcfLGYbmDzUDa(S&EhV~@M{!QhF45x{5aYT!%grIJWA zCPZLjV~TR%OaI?Gu#|*;3pE&Gip%(QYCabRSzxifXbFW_h<;~DMV9TX$9R^kJ%q(9 z@u6a&!2C5jN~P}Np~q8{2|rt1E-=0WyZ{~6i^y1irSsTLlD;u-T+A$j?`35ysX`tV zKCmt{=gq;VrM1+BD%nZklr7&?k8W&iyaabcEIee_)YjS=qBu)zIMJ+wsg;Dr#y8y?Sd}L_|a`5OLv$fN~ZGPeRq_V#(rkH!pg4JX6Rf z!J7{)t#ga zG7j*7mtA zF3`@v5`vJ5$($TYm*D5dKB^KD5*pgu`VMWhB)qmU+D2Z|;12Z__;^t!K*0+D!b^wN zli(XM$F|m1>x><+UqNp25)pkxTl{3&JTCdlNY;T92{Q{tqOiS#zR*jn_@UtTp)YxQ+hm_ON!MMC&J5 zB@nhx!-kuz=a>Q;?N%C9Z!VA70Yujtpn9Jv=%!}+5%)X5@yv*|EbSF+N+ZGcRU2##>ZtIrL5dNliimLfp~gMcRW*pf2HO zY-~K280K7tU!kO=R4-y+nhnu@gyta|fV`5>`8mPI%kfKe8LVS&!yN1JGhzL=b?;t0 z6DH~YpbDPGCYO_+E00=+nMfY~X;W|fcVx)6$TO76rp-fFGL#8r6I~W?+(vNu9qFW5 zxqcF*F|cN&=x(Gkdh$5(`-ebwd`PNLK|kX!5duFtZE^0^zS2=buQO9(@cf={^{Ixz zlQ&?H8VtoCsWUAH`Fi0Mg+QvSjaLdQMG(Y7A&r+I9Afb(lm!A_E-gmyx97vJ@rAc$ z3=0c#OMYxmfD1`rI*CBG%jSiskrK!>(~v10@h`ICKhk>-H63`ld!b`y8CLOjyu~y+ zTRM>k2Wo-VQJ_tXRZ@qH$YCaoDM^aKGwIc<#cxn66J)JG@R+a2Q0c9$%I?kUEh;rl zyW0xInVz>WMVeRF3SBe3xMiqd!aeA=ud`@8RA33KstbqSD&Qd}ylsw{Ml-1=nExIU zlVx0zL@C_DqEiK3$qIv5s5Fl;q(c@STeb#)hnQiQj2?Y07QwSC1YM{>1fb6p2Akk$b%)Ctlj*uY|?Uv^0+B}1ob5uvK`7{1aaRyK9u9d6w zw=m*~F9IM^86=BMZYf%EFX<;;_kHJ&^D<=wpD7HBUxRNVjV*&i!5@i0nEJBF=9pF- zl9ZK)hccm>PvS*^Y4lQ@lLWr3c#H1Ba^xw$Si7oIv3w1tZTdprZz|*e(}b%e7!IXDZ~1out8+Grn8ls}^L=9XIw8U8T`c_~WXu41~-E zusBs%h@KkPJ_|3rJew;sCuUS3@F($_k9+mG`gp zJUrzwvwqvH=HfAWvUU%IgT3;KvWkvhoRteFd}2O_S5zwya`_UA^*e*7UP@ph2-%i& zfo0uXd;AN={YOssT*wD9;10rfIu${g7(#{q20f%HA#ij4TymBW9K*s5LY!s=@18|X zO^rG>Hul%?F}rr9QN18@Txu$#h=|B`v{_buJ`DysdJGKa$jHj7xuB(~Idrly&^Ixm zLN+lm(NV5A@(8{SR_MTPr$s{Kd~rH)5@J?;YzDHYRj^ZssX@F;c>40qAi$iP*E`wC zt*x#9xpM86zbAq}#VU%55rPyXB+M$Jf9t?G6kGE-Q@Xx8n6oQpriyaHr5! zo8M&zqJuUS)SuB%PX!hST~xFyRqU$ec%tV*Cg>Ui2$!Nf%f9oynE~n$Sxd_zI9B1Z z`|)aTxuNoKwM9Eq=75!hV>09U4!8to@B6}^oBi3H;&OzF>4GdnAlfRgrbewEdFn~? zyB^e&(^Ia@U&i2xc#~OT-hRcdDZlBD{aw^bO3n|4Um@m0Q)NSC0U6_ZxHz>E*<`c8 zwl`f6E*+JYPSo*nuONB$N0e&bo8oFgb?ne|IfU@=aLsVs)%@2VFEk7@zxiAVnf5=v z-eUJJ77=~8o?!G$a%==aIEM!@VopiaQT!-vvIN|!sDDWqLL z%2G4CsHmV2kN60cp{r+c^+&zU-1V0)m{toFnHd11y}8fLrEWTIzoiirZ0>iegCmJq zi;e!sXRXApjQ~;Ra0ia;p-jJv>Tt$0EOt7;e$42p<$y<*u9(UtJysNCNlQ;3URYST z-z_x{__HFH-EX9?EMt65`kt2{kM`QGHrHFvDFMXh{b!d7w>>lzoz?_*ioN4bZL5pO z*f7l0(=B%03A{QojEFTay{wci`aZKnL9NO+nc%i}4EbJ*TMT;Naj!XzBz9 zT0erXd;uSHy7uw77=-na#~c^I8}aliT3b*^P!RG9A@{P@zZc6~zBjINii&m3relVf zMoQ!>FTnyOcTS}|ZnA(B+F~e7H^dS0K(d$CQMHf&_E9k>a)P#u|v+sH0t47a+ZzZ7WJ!PVYS& z`OEptNqZ>=v}Xhzp}unicl{``#76CAAmdui@vPe!r}hu{$E^SR@v}h4w7i#X|I-(( z_l*e&Kb_N#tu4XDq8kAzOcmE51B=3ge>M`pPjE24sL?;ut^inX2EYZ;>MohVVXbT6==}J;4g2wY$Rd=hEx@LN? z%Rq4MWJN;1{xm7deCL4D?{3CsyKZIv`oH~mMq)fm)Es8E>E0^#LX3;k_eglIEiMIK zvf6*Iy#NyYfNaIOql=7Ib97AbpQEz>U4ll&!2SL7)6u&;E?3i_|1*bz1|xH*nzegG z)MainJJOmbz`Kn5-!Ad^v*QXgBwrFkO0P%CCJdgn-t6+PlcG#uOnNDR%|VPzcNm@& zQg|!>-d#0vqoQ3BG>808^Yo|u&YXtQl2_ZFaqC$b;5EuIdZxVynxjJGqlUwwABq=A zUrDI*bLzSsRg^)~{Qqr=%TQhv=fRsD6Y|Y_1fWGp5gxGIwf~da*^436axI|J%QEAr z2oNTlEz$!dWv=%>ZwrcS|0VSD`qjUTiM*KsS2T|fe3n!Iz69S;kCiETDmA=Wu+^q; zPvs-{>@BQBFtcc%Jo(P(NlX-|Nr}iu^;S!jHP1aG>7w>8GQL(s7JMtD{C6vGFd+Uh zU=wlHy&k?}8zyS!BH=hBpn>-rc4CQf0$^NMq zZlD$iFKFdIFQ}0v@U+BQ5}CEW9`SoH?a(TDRt+zMI8$k43-VW9`)92`iug~6P$t4) zUN(|XQ+CMm`iqoli0NCE6rLpLzWtUVfwSx?0oyvZ4+Asm`_HO4$sxVCE||7c&Cyd_ z_3Ba$lOJ{(+Nm-tm^qa%xSXc+^79H6yISD{N3K>7%m(D(Y;w@S&#wQ%#}gF|{%lfE~i75ZxUMAy+-o2GqP_WtL zcZ3Gq?Wy?3!BZ+<;n%O{u0hKDLd2)B`Dn=^JCRJty@>4-2$e5{J*!%8_Dh*WD=OJl zVf%SKE;N(;NIHtEzD_yBd63Dk^_MHBxN?Imwg!l(BU@fVv~=#eT)$x$jlzc#1qB7~ z`E&ZqpZ*;N9e#$VXJ@qB-0I*`pyB7&0VJMAp6~$~W){!Wqj<3y^qiKXwJ@Wa*S|`x zCD08or2|hlNkzjvxTjhYv~qfndh81GYli>!{~l!z47Co5AmLMvi;ri;v1$8@|8cIO zu~8?>^B0D=%@Zp?V1ZuPfzM4B*0fVa2aT@u0e196(trh7EMlp| zmY%ZHO9j{8wmPPl!)cCQ))gJS`7eID=}6AM$C`k={m^a84w;(DzcDKDEKt*4gGBzL3Zs z0r_$Raf8cFg<||`Mo&6C>XO8{cHfV33JU5n+U0t*lEIIwLDr0Kn25_OmceV6cDPh4 zW7g@nP+>^R#3cXQxW!!w-0k$s4GxvMoj4zXU{r!duv&DmP+16_lMh3a%Xx?NQQ1hH zbJd4Ee<4+XgHBy(jG|F+w5pbX${yf8=j4HRy4%n7>9IV7#Jg&1`1Ndeha&*q+R64e zf-?aPTkXEL8snw|QN^6i2?@{O)Oah9MfUldsXm_X({gfhT6fEXq@4zk8PhW}i468j zHPZE@oZqyKe-Rn^x_U+-@6!eF+rMncOkl%`o8#&9>wudpfCX3ucnwB9QGgCX6E*lm`v@MVpqQ7!5hV?p*$JKm4 zwzB)kg`Y5crff%_aMO&&^g1I&RIUfQDL)g=EbnRFlQ&V&iMw;Tdne9krjFkl<3o@@ zSuTt{QupY&!S{lY^AJZ8)XR6epO5wg%czz~v|t>;-vM7t>j)SajS)1CJiPF(|LsgR zr?FLj?FSK~%J@m8M~%F%+W=lL>$|X$0>1N36!DRXL_8^y$3G@6;(N=ewM&{M53F^% zT3q+yMzckYIkx;a2V;jH4gcy3-XBd)d;-O#^jCt_nVCmoq||sx?dVhPD#p zuRKdDa*LUTOk?hS-!1y1OU#ledUak;i&DRUzoK=5a4Ad`*y_YQWNZ7aVpU)0`~F)dGH)Rzx8NDEj-Mgg$< zL-{PR>dIFCnbj57V`fiB=GlAL2F>CMIrG@W+^- zMB*VqYq6af$2f|@@1xogk%z3$zLVxQ3*FCbWmN@EE#zK-QcN}vV=!-tKM@q7M)LCV z$WfV8o*8w&oR8_56_MQV_+u0D2z4By?>Ha7%}h=ALYhKdTP7|Lt|1jJosc*RFTOtQ zwH3Ygyk#tJF-Q+vwNHMb1!FUAXPGHfsig8G>f!!;R!uTRNH|JpgoSwO+(3me!E`mv zURmdRFpfvvk72Bf|E>2NFt{V?%1iykLcJFUP5hx1J!zKk^&sVYNy+S^+l}!jL%?OW zOo>HeK|mGe`W^e=8x~?HZiXoPMcZKv(-vX!$iYE$X=e?fMU?dmL*@c;y%lFq1U;Bb z!X*)iHtB*1gcs4WwA_!^SZggN2>AovkP5Bhqo{abRr?9z>X0PlNZbKtz)i=kVk<#5 zx=)-bYpv;^;k~)w;kM`sdPNTu=IuL3P#9K`m%Ws9$(;dBOiIF}D>LDx>u%KdufjFl z0q}51uq+)LnS-F?AXm)Gt$ZVo6uRd4^L+5Ju|RU%&ZP8O@-*hAJ&uZ(#Rx`4gUxoh zC4ZNY*J=J2rl_3Kzc=SCzdsGJ2-0I_jnBVZb*025ivODH`Eojt8!xEvpj(V5%AKm< z{wnEviSaw#g2$^wj5x~Yu_<*(`UD5=TSUq*MZ&U*3N8CS#oMcJEedm+{S1cx?1{k6 z&A&Ta#_GOlzxieeCt{McsmaW5hi1+m(d6aK5=zhOq8enCKMsv=q4$*UC1~5BoV~_P z%it)3($dfjZjYtgnDl$DU7zjPkj~hJKnl~Pbtp2?O{(2mbY(Ubp3G-;6=(yFb*pKs z3oqS1TkE-x4uxQP3Ejx(WpjQ!qd^T@@H63v^{$p-kfYOxiW>WJ*qp(Q6kwa!*fC&c z&2PWk%1}he(&X@SKYvplZ{vJ$@m&2Z|6GdQ>z#C{;jf8xffg42l4g~sQ<=qP$RM{= zFH}ynUu_@3Z+5pH2B{m*pXR>!%Wp$Rm`XN3+0kn*RquTsD<`k4r8JT40`>)UapOvZrd-0T zo@GoCqJ^2)VBa<19_+9nh2*IxO75(xvSK~@aByQ9@!L`{ctzkS=J66P6)kV5PC&(I zlTmX17!A9CBVFITru*{gV|q{YJ52g19g`?ajQQ+{h;K4d?wiZv{}k>KO?1l6J&Trc z1_ydCT5mSi)0UKqX8oV5#I#GS5&Yz?0_v}sNp+TcREkO$SLe@PWk3t^^UK`K5dS#oE$7isYvoQ0RuCSE5a zrg7@|g^heWO|Fgojt~pT7lf*xez)aT3ugCfFoS-LvZ6;Aw){y&-OyLsQlvVX*Syqn z<@gy_f1BvoaR(MRtCV6-uSQ?unHHvIM}7FZ+y+9DoSYn-6%Uvz}?*8t2KR)SOLysnA7TW`l4vrzZkN-`xbMfbM%Zj+kHjIIJ_)P|I zQ6yfajX>YDds=kFqK^bhnR}vzTY)z*xXi4~hT>>>L==+QG_ecZcyUowEDoydqv!%K zNOXcH{i3ozv@CiO>e-I`HPxK`bG??l|9eewj5*Or0g`{5Jb)bgBIQuBvkGECVQ|5o z2(j?*Z~b(D+Zd;r-^`tSy(gqwAd*I)Imh)Iiqd3r_il^4?0f5$=%`p5pbjYi;Ku(& zC(?p3F*9#<20mh^yfRx4-N{eNW@;IfXf@3jkmBmUL58fF(jRt;K4N+(}=0DkaM>lzU&x(r87w9q968dytwb*G9l+7`oQY4aYkq#68s+5$uuP~g+H+G`KR-YDL#;}q z^7p?4Q0>l+0N+l|XJp2Lgx>b42wmYX8_7YJRKjIKpz&w9gRra`LD;-U!aBLy3>p}T z)o@FfwJMBQs`s(Im`Rgy*(q(R?pyQ0!Z9Wm{NppLNTURMS#)Jjh8|#5ap`#Ydwg*B z>KP7Bc{h;!N)Mj1>bWi>y&b)SCtyqu*Vzw@aN9_Ni}m7@SfZ-NztR48d381m<15W9 zfG~|W_xpQbJaT*7@iQlk(enxQd(=<(or-)b~SGPN3cbstUMaL@*^VZ}1eIVT{Q{@U6yYmV@j>nuk%VU&pO3p}0P=NMwPGO9ytK%!-GF_ei;Q7 z6*EKTa(7CyzJHRs2kkbQec?5Fl#Un3RA0$R{{+J2{#qT)mr2P+wBb~Kr(C$qy8ow0 zHE;9kWMjMR5gN9}IR{yh3e1sWl%@|2qR_>yx?$|`5;rNPocc+bAn}fU^!Sb&0(hK} zf}UsS7woQ~;bFC1KKqrT*4kM!G;|s%Ok8*_2omayb9j;r80|AZUr5No)M>B9ND}Ka zVQwD#MtnrYvFrmcIe{j^>c7= z$p(W5K-T-sMs423C&Hb5=ZzQlx5vqF=hAe&jrQ5&@gMu*U8Rwj(M^vKCw4V}#Msa~ zbr33)vdlPPsngX%L}C!$q?ZeAO6$`x*2gU>D@iKcK^u&D@)U`kS&_yB2Lzv1Kn&Eg zVH-xK^g`_AeK<(e{s;B6p;NMqlWkxReVdeOUaEQiA*pm*AD+hJ8gyz-s}QTJMijX1 zO-tYJ_}|MMHEv)xosA34?SR}gvT4Q2i@{S_a)LxCOic4!=|u;<-o3zm6(uecKE`5r zz&m}<&?MLpefG*WW1kVFohBQ~gcgl%R4H!2Z57B4hZ|peKM=VQWR+R0B7>VKB~rla zbhA0zV21mdu+ozr>!rxfKyJoqZ%FbA1J}2oiY_0l{@xNm{%e2u~`@ z+P&0`be>YfMT(vu)$(s>!Ha{Zb9e%ZM&hJ`{2%Xusg;&V{&>m6cKyY9hu&Ehb77;w zP5EO@`-Uo`F$Tt0BKmq>o?jmN^xHVzc(I9ufMGLx>LdklwgNh5C|@pKW0wrss;^8b z6CiSn1O9gVwVfHjXghcTM&v~mlLlMi!EfS9&Y-Gi>u?1KXpZ?)weqGhoY9f+yA>CL zhMkW?E^LXti)oye(pCJQg&CNCvg-HxIoj}YI<+Qppv5-Ypec9sU~%cW%$=hxYr0

p|(q!GYeGO0HCUb@rc>XY1%<6&f!F>7o2EyQgg$UXWpu>U!QhcSMB1@Vlx3r=RMKDj@NK=CzFDNzXxZ0ZDCEL~>b59d2wx%Zrzk z?WA-ye{>1U2ut}|C`pX304d1xuAsH%M?Y;XKc1y9BBs%Xt^f@^$^=zPed`cchmMzP z|5Le9U?Us9d8bUbC7X;QQ!vt<2kF}<5_K{E(w;@{5hcW8jM85#G?N+(ZJXohYING5 z5fE51v%Kpyv;GnBMGcmQIEIi+DTS?P4v2O_{W=KSx9ySI5lBG~ssBk!!qF=U+~qwq zcG+~fweBG%W$s4~caIRi!(M$Ph+y)(`lT?$8Y}-?kLGug_M_jG^<5Yx1VLt!c$8)h%mzng(0kyow@~;lZS7`h}8l=Tw3I~$a7*%Sd@WmvU}}a? zIKX0`qErIHs2sni?s~w2w0P!Y=-;A!LuXmrBzVrWD)FG9p*2!9Mn6m3=+=dSP4K$z z5ZI8uH6ZE;HT?6i&A!1G9!*|!zbnyudC})#v2{PRHcXJjeC=Pi!>p#JO52u%JT&pfTwCJL;#K@W8C(8Ta~WjV+$#E3jO7sseO*Q^+{0Zo4H1 z%^YusnMyufZR4NLt{}(_+Ab+d4e0dTdo20WNeig2xtyl0Is_l7jV9dfT=krGeUEo1 z&=DTpX_=6{UGmdojjj^AkP`sWT|$UR4)igjHM_`4ild>2sT5Sk@tyna(HTnjC7j_;pueT`l_x(%u@ctX<1 zDmmkz?+%ZK1pJsTQn&OtMrB*K+;7(w>C z*Qr})fP>YI$Qc{J1}u`zI=@1^iKJm1?jgE?mx_O$)t$27LsBjbNb`jqc6TqQYAP;% zpi%@ONCaqG)CWZMnaK?_-CM`r0BI>*WM7};rW(&21BoGK44J_Xq|E5%Q{?-Rxo>g6ku1NJzueFk(( zx4hD(nL7lYeqy`4f=4(_nwlTqs;mq$KtkAsMkm8>-W^t;N~|b+i>V-&-DB*<@&=dNgKu@$3}- zch%C^Ptn(aOOO?)(n&yEkyo86UkV;<@%!tY*pQnm@uR_k9aIw301NR8QNPyHbuAqo z2D=}1vB2=Rz)%Zx8K4x+S@Z-y0T5ktqqX#Rj`RW$sv=h0Yt~A7KHYbGyBWNRNfTw- zZyQ9QXV2dr*{F>g%s-w0W}6veQ-e2OTfg%f2%uRGMNBd4N}mq88o#(*)%BjM4j%O_ z#0S2T6ol1OfZr2ZTlM1^-GBd}1HgH-apaA%+8)cO9Lg86tx@pv>bk9-t`sQtJQ za6il?3YZN)({1-@y5=92Tqrb?2!~mS-lpE%{V5c6^Xk&aPslMh5AL_n71_31)*%Vr zI|1_a(A;70r_432fN*l1b&AsY9y38lb8}PGw%Y5UJJUm|E~NAAx<6Ih*sOhny(E!s zv}duv`UU)vZzXppBfU2I%OA>ijeL%pc37NeR}|&so}!ab`T>i9^PgQ+mtdev3|E^D zd{}Gsf}1kkNUz_vM@L5klOl`0C$M_BT6KNuQ1smMKRXD0gF^i}GlTh35tfh|!e%-1 zqxc;u$?;~g)?mEL-Q3xaM$7bPA)$Mr8TZg<4&r};zX{d^TO>Z$CXZ+OW5*R6U=Gd- zAu8hIh_=4|{*pYPh9nBOZ0l?mHHBd?0J2XV(%^k=oyut%JRIQE2F!@szcx1JQq~4z zU(Pie>glD`SJOY9K^0L*OM%>pdYO0C*>+Z0+on zIyHi-0NE{rCg%d{dSJ^?gxel1W2pe|$57oH^n7$MgX0}ULtDGlsfkK5co6V+$lZKa z(=??*^DAyv<(jKw%Gn50*BZ<9)aN^nH!4)Acz zl%Yvdqw|-MtyC05>uP7oa*UA!12d8}r7Bz!in~IWMC6m3qfH)K@07kUdxe9bfv#vW zsh-frI6L38Tbi6Fz{Jh_tx@x-_A_Ch_44~JMkh~+Ge8dgs-1m*n4}oKFq*NwPZ~M& zwxU(JDW{Zmk&i`0UyvQJpk?K&p9zvQhf8gG;G8RO^o|AU;1j$mM9}3E${#i8pi3V2 z27XzNg0_Oac(TH|{HQzB{s=v93K^qlIZC*C2CaOFydGe{lc_u&J77T`Tey9VCES9QfUKc=!bp!lRF)a+!EF+D$kUWC2`;zGnM@XTiFSJ}|{E z;4-hWjc&Cj3?uyHtcD24)L}I15P^H=UE2~p`P02|?&*(Kj+@!W{%oC|59C3wgkUbJ* zL(ranr0b3P^dP65j&fxtma*FW@%K==S$se&fPk-RE11#OO7fhnCFHEL{^pV7*f`#; zSrX*^cJ>Z5pD$?>)CrBIEV4x524==xNw5adtQtzL&xJ=vgBl{2Lf8c(Rtjc)Xsb3Z zr1%6x(tDV!oL&aaitU*VTicF#=xDau*1>PpxOI9|96BR9(}Vc77@Q$Sr$#FQ16LL--awOFNP+t)MZGlvul zE4EACz6vF;Q12cc1;j;3qa#W|zaw5bzVZBLWLr>QYk~EJmqk*hOUUE33LM3LBjc}> z)?U4OJ!MV!LF=nXx$uNb@ju(8jc44GpQwbuWox1Uo~A!tC$8HJ2)Hh#C4VC*fv7V6 zb6uEfSZZ$eZ{ZEt5O;kl5u%Zdy%1ycgdlZO-&VaDkhduj|GiQUh#7$^zHo2LC(%Ix zWzv>fpojF1rvLdX08iS&6kYvZp}3x{@eL`sTM4K3r%_$Xb&~ztD9!eeooYTkxWiPD zXo~UO^su_iqs_eEg&BIRp~oT1$|bmB=oj!^pTiDT-2D5L91OShnW;y(FT28sY3ee0 zorP=3%9fK(cP2kG*Nt};af|*DMJeseWp#@%8fY{8TrDX-@9JrM);~y|b;Ml#!uzI5 zDx+0@=;ZUO^{Jb*R}vnd zU3pq$r5#hYI?=J4`UtF9bK0oeTaY~FW04Ia5%fX07+r#dI2Es_D;K4vhUrSDO%=BZ zT5gFS!iCygGTb<{wq#(>DKUFW|4N5YRfvF`7C4!~Hkh~w{5#VEwi!gy7%#zkwnuD zx8b*KD%SWusyq_S%uTWh>a%EAfwI7D_wT^l$Cg}eBoR=Dy`Mha0`|b%bkptrSo~r@ z4kDIH&J7qr3=MtNagi)x3r~kP6ig5Lec+`9^dK8=e_}n`sSDB8^i@aoBlsp9OljRv zN>}M(1@}TOIgW}=nQXFuMUZ)04Bac;GZue*gY`+4y~9 zIxs1>xc`m=7M0RZnwpx<`?HKowHDD=H#eBW;z-cNMV+MvhirHeud}nWA4`4`s|5x* zwQHV`W3h3WLW2R^WeMkE%--b~j4&2|c09u&0G2_JPJ`6u9kwyj)SccLyc7W+?9#Aa z9RKI+Ml3I7q7p!HwL&!cYZ|`Gi<6wwQ|A}J)+2neXjQVgA&-MX zp~Pld+S<%cL1J*f4X{I5XEa=nCzeTgnbb5KhPGNC>kE|Ky$E4rXs}FCwF(a^KnLW5 z*1_M&X;W$twO_*rk^Z%%8YmOn_5%(bf3*NLZ5Jy^!^x=%&j$DQ_IQvTfZA4Krmd!y zi~JkM|LW7#?d=FC)BMmGfT*UUQ-lly^>OKCI@P&vQX5r%w)@WQ0Plud5j@ZRZ;eNs zaf!B65OGXeV|3~)PE$&I!GfzMiEqM~S%Za!!U96eV+5Y$#7H&cMp9XS`i zZJ^r}fFiqX(%oSxK7bPAU8s_Wse91n!A$gMFqX%EW_OuwGRgFGJVcRo7kTjAOEwK)+OgZjBPR zrZ<0L`+502D}g-x5ejx`D!gDO9$>_7`&~Vs?NkdIP`Xc+$zzig0k1xPRNjDsMK}mb zrM`;59dmH0x~A6OAX-yTj|oT@G2iW$=TrW?7rX}o+z7DUkD2K?@H;=o!K(WS>=sC^ z-*#G#Mip&>zxhVUxa&G<1lxUiMP@z!6YdfFw2vaf$yQI)p^GyTF5B4&j;&VzXH!)l zkm7E}%84<-&DGxQ2hnPeN$fYA4^? zUUASs?con5lsT{k-YfY(NRJ-PEe#IJ@{CL~l?u88NyUw51RpF>X^<1BW3$G?R5r;S z-k&R-yIKQ&G|h$rH-V(vD7@C;{q$j;`3~(MbV79!9;zQbRZ8@;AL6&`-UGAc)}9w4 z*F{V18|mI1YQ+B%M4-{j=QCS?=&5<#&Ckt!z-4Bl_zj*e>UwWFw<$)%KRm0y7HcCZMDIEvY9*eIlhkI=!truozejDO*pO? zjFp&Jh41~7>R(=Xjwt^C!3T$~p5~SY-ZxCobuF(O)%CV1O3n4(@KOgGz7bD_JBZlJ zG2>5%tv$MXEoc#Y04xQ*-H%vV4}Gpp4spq({PDR<@yi{8I>Kvy-?Q=VAk#N(1*F31 z)PU7V*wFSfaDM+le2NW*2x@D(YUx+a6YNyFnAs{5))XL{p9!eq>=3e3$2ysXsN)mk z4{wqfbGDWpwm5mh>jf^MJfNNTzh^%Gh?xcM<(nFh!fGL#vs{JC$n$|jW*!#ZB37@= z4ubcCI}&C~mG?c-j+gOE=2B?uI zT}C8n29)%>8pPKT@i@@oXdt*Q8A`QAb|-&hEWdCVMrKS_cLF^!SVBC?S){h=Wl=3U<6@f9q)WJE_s(f;hw35uMf3I z2m@o&`3E0nK*{xIG+5#_yj*1gBlvsutIJ<6lM8;oG6n!pDFOzJ?^2;PzVh$Cqk-#}h82I_O~1V69@a?Tkz^M<+ERtMl_6FH}%= zYU(*2BLSsM{81oSIhB#P(d%Pm9Rsw&eI_*>`DQg4J{{T4UHGfhT_l7Q`k1plNm@@= zvAl4F7aD_>(V(T*p-o#YE3(5zHS@5(Y-d4)-f`C_uz_fxQU6-D6+lAx?Ozk34R=;{ zBiFwA*J}W>JwRDj7j$(RFn4KM1z=Fe2s2HvA5$EGmnie6f{uq!tJ-}PkSlXb^t^6% z7BhJ?kK*CK)CL_zrM4l57Sccq>feBK`KdNPw>o!i4ZPNw7=D&bkVly6q+nH1kHg2F z&P0Os2lL3G1AMO%x$;k&FsMM=Y1zMDa$~6oUo0mu!GC^xJdVfk75|nGPTRkVicSvr z6jI~Bm;O`R@PBo3_Ww-pZyaB4$|5Uv=$yio6k6q;+$PjcG2ysps7`K9Be@yLYGFm_ zBzLu~LOI8%+$_h;ZR)70q~_j1N}||u99u2U_0jqM1K%G%kH_|YJoeu8xvuMdy4_rH>OTirC3QRvgP8Bejctj?< zW#>*}1NSK4G$lE<`Vxr*J{mt<8=i|OiTPYSa9gx<=Q!*Qi+7=HdZE2eHnWAcFe360 z2Bh2Dd^gNXtPzXFuW}zGmI<#ac`n$#=5)Rw@}1}rT%T| zU3QBijNs#K|KW^`j5i3Z4cyUi`R%D1a161GoxGGE=YRi~B1cEZFmH1yi3tNK9bMfB zZ}U?>b#&o*`3{4Pr|RV7ePsAfrB{WXYizFL^yAmH9+RPM1$Ehg%DM1&9=}J!jnicor}Dc@v0#R9e7MV{aKz*Fa)Hms4bX%oT6RNC*Q_{-+-QsNbK%1@1PmTw$7x>qA}5MF70P zsAfDufP;X4PkDJcV--8!>M-LSr#C3~7%S14E$Rmq0|8dZ3$$AlkNZW;@b;?j2yk1{ zN>}=R6JoO9HGRicJSzL}1MEV<$k&+!XpYq0ovr51ffLXX(6q~t5u`hLQS#0&%lLJ} zVURx<#ZI~*XoBC5N9GmM2XBoELfsEGK1#w(G4N*sC3!H(Wu9T}% z8l|83erOh*P9I$*(}XQ^xpq76x7y=_UL3a|%9y-BrRzl27B$v5^QGRNA>lc!9i!@E z81&&&^jhV(cFY6>b&iUf+U=zCEf~A?2qZJ_<8!R_LGc`j4)}-!C{gCm>rXm5mc2#D zD#1QrW)&8aSu9pL$2PKO`@N11jsV4ma+iZa4RV9uhhiylDek! zDZO_|>K8}e>&EwNJGAfBPxM$gQ-uS9Hv}Yn%xvSr>SR$b6~Snhml#Q}1xlo(x!4+b@_nu5WajoG=Fq20>`ZMaAHj zM8Gef^lrQ#C+a*)wiKJBt{=Cvcyu$?SLvb%RePiB$FroplnHzFGG+8qRjZ=bD@>?T z!_?(bHO0U*@ICrf(=qL0Q19R2GY9MR#_)_6S^E;Wy?mEsfs zq3?u0lbMK`Rp*`=>+`ESVI&RPbHK*bR8Vw2qS5@(W8u;~2!YgM)i>Fex!SCzJQvy| zAl^Lvm0Xwn^6w%PG44V?g-ZpJRAW_s%R{Lxv=Ju|(G{rlt{~o4m8X-yUMg>MNCg^XPi4F9sEf%pb!-1Pb`Ikl zU#{Dcm<7fYpsPI{kIONZwXb!iJDvr z$|(P>*u)ve!^Y)DB>}EmQN$h5niumo@VfUhDjH0*kDSjl&s*Sd_GzIppZ8WnH{t->vE#^gFTuP5E?%3f3-}3~rrL^jN!6~Z z@T9YYbwpQ&#ZBAjn5;tDT5EBPlLIylv1USRJ^MJ)Oyc}K>TY0VHRy?5f~LvD{tT@b zg=6WSolw~ZR$#iy+`i>pLR?Xm2CTCJfPN`YGaFz1O2Xl8`;fvh9-nG{4M1E%{ zmP)0*AF8jJ>IP888*h^uLW|s>FeRHGm&ffk(pglD#n$sjKMh0+{$B;fj1evOkjpVYSSn!y?A5BBTGp733f% j@u#Zpe+$>AI>M!%ey4lZWK<#;C87vUu8wycf@%K(KlHNw literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/grad_tool/level_adapter.py b/debug/accuracy_tools/grad_tool/level_adapter.py index 5e5e0172e8..51e6717d94 100644 --- a/debug/accuracy_tools/grad_tool/level_adapter.py +++ b/debug/accuracy_tools/grad_tool/level_adapter.py @@ -30,6 +30,15 @@ class LevelOps: return_list = [x / element_num if element_num != 0 else 0 for x in interval_nums] return return_list + @staticmethod + def save_grad_direction(param_name, grad, save_path): + if not os.path.exists(save_path): + os.makedirs(save_path) + param_grad = torch.Tensor(grad.clone().cpu()) + is_positive = param_grad > 0 + torch.save(is_positive, f'{save_path}/{param_name}.pt') + print_info_log(f'Save {param_name} bool tensor, it has {is_positive.sum()}/{is_positive.numel()} positive elements') + class Level(ABC): @abstractmethod @@ -69,12 +78,18 @@ class Level_1(Level): class Level_2(Level): def save_grad_direction(self, param_name, grad, save_path): - if not os.path.exists(save_path): - os.makedirs(save_path) - param_grad = torch.Tensor(grad.clone().cpu()) - is_positive = param_grad > 0 - torch.save(is_positive, f'{save_path}/{param_name}.pt') - print_info_log(f'Save {param_name} bool tensor, it has {is_positive.sum()}/{is_positive.numel()} positive elements') + LevelOps.save_grad_direction(param_name, grad, save_path) + + def count_grad_distribution(self, grad, bounds): + return [] + + def intervals_header(self, bounds): + return [] + + +class Level_3(Level): + def save_grad_direction(self, param_name, grad, save_path): + LevelOps.save_grad_direction(param_name, grad, save_path) def count_grad_distribution(self, grad, bounds): return LevelOps.count_grad_distribution(grad, bounds) @@ -84,7 +99,7 @@ class Level_2(Level): class LevelAdapter: - levels = {"L0": Level_0, "L1": Level_1, "L2": Level_2} + levels = {"L0": Level_0, "L1": Level_1, "L2": Level_2, "L3": Level_3} @staticmethod def level_adapter(level): diff --git a/debug/accuracy_tools/grad_tool/utils.py b/debug/accuracy_tools/grad_tool/utils.py index 195297c5ab..8d0ed36ce3 100644 --- a/debug/accuracy_tools/grad_tool/utils.py +++ b/debug/accuracy_tools/grad_tool/utils.py @@ -1,11 +1,11 @@ import os -import time import yaml import torch import pandas as pd from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen, create_directory, \ - check_link, FileChecker, FileCheckConst -from ptdbg_ascend.src.python.ptdbg_ascend.common.utils import check_file_or_directory_path, print_info_log + FileChecker, FileCheckConst +from ptdbg_ascend.src.python.ptdbg_ascend.common.utils import check_file_or_directory_path, print_info_log, \ + print_warn_log class ListCache(list): @@ -22,9 +22,9 @@ class ListCache(list): if len(self) == 0: return if not self._output_file: - print("dumpfile path is not setted") + print_warn_log("dumpfile path is not setted") write_csv(self._output_file, self, []) - print(f"write {len(self)} items to {self._output_file} the {self._dump_count} time") + print_info_log(f"write {len(self)} items to {self._output_file} the {self._dump_count} time") self.clear() def append(self, data): @@ -56,7 +56,7 @@ def write_csv(filepath, content_list, header): def make_file_safety(file_path: str, permission=0o640): if os.path.islink(file_path): - raise RuntimeError("Invalid soft link path: {}".format(file_path)) + raise RuntimeError(f"Invalid soft link path: {file_path}") file_real_path = os.path.realpath(file_path) if os.path.exists(file_real_path): return @@ -64,7 +64,7 @@ def make_file_safety(file_path: str, permission=0o640): if not os.path.exists(parent_path): create_directory(parent_path) if not os.access(parent_path, os.W_OK): - raise PermissionError("The path {} is not writable!".format(parent_path)) + raise PermissionError(f"The path {parent_path} is not writable!") try: os.close(os.open(file_real_path, os.O_WRONLY | os.O_CREAT, permission)) except OSError as e: @@ -83,18 +83,6 @@ def check_numeral_list_ascend(lst): raise Exception("The input list should be ascending") -def localtime_str(): - return time.strftime("%Y%m%d%H%M%S", time.localtime()) - - -def make_localtime_dir(path): - if not os.path.isdir(path): - create_directory(path) - localtime_dir = os.path.join(path, localtime_str()) - create_directory(localtime_dir) - return localtime_dir - - def get_tensor_rank(x): if isinstance(x, (list, tuple)): if len(x) > 0: -- Gitee